2024-05-15T15:15:47.608 INFO:root:teuthology version: 0.0.1.dev271+g1ae7ad8 2024-05-15T15:15:47.609 DEBUG:teuthology.run:Teuthology command: teuthology --description orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} --verbose --archive /home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165 --owner scheduled_gabrioux@teuthology --name gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi -- /home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165/orig.config.yaml 2024-05-15T15:15:47.656 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-15T15:15:47.741 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165 branch: wip-65480-squid description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: gabrioux@ibm.com first_in_suite: false job_id: '7707165' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-65480-squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: de5210d58560a4684fdb5b238ef3e6e0285a1de8 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: de5210d58560a4684fdb5b238ef3e6e0285a1de8 workunit: branch: wip-65480-squid sha1: de5210d58560a4684fdb5b238ef3e6e0285a1de8 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 6602 sha1: de5210d58560a4684fdb5b238ef3e6e0285a1de8 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-65480-squid suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_de5210d58560a4684fdb5b238ef3e6e0285a1de8/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: de5210d58560a4684fdb5b238ef3e6e0285a1de8 targets: smithi043.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEV/s1jhomw6GouUuRgv82hZgpb9i/dUDfIiPcDZ6e6p9Sm6A++kvvAuToPtje3gSPZk95DuIt1iFCeE+cblstY= smithi053.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNW37ucWkq64eR0bVE4G0GhITfumYcKeYmmwMjxzF41yJvjbxq+X+l6WWaq8t8dD58kOevOQU9kV5mow7L3cPZI= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 1ae7ad82388e92a475afff437d49054826c019a1 timestamp: 2024-05-15_15:01:21 tube: smithi user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-05-15T15:15:47.742 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_de5210d58560a4684fdb5b238ef3e6e0285a1de8/qa; will attempt to use it 2024-05-15T15:15:47.742 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_de5210d58560a4684fdb5b238ef3e6e0285a1de8/qa/tasks 2024-05-15T15:15:47.743 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-05-15T15:15:47.744 INFO:teuthology.task.internal:Checking packages... 2024-05-15T15:15:47.773 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'de5210d58560a4684fdb5b238ef3e6e0285a1de8' 2024-05-15T15:15:47.773 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-05-15T15:15:47.773 INFO:teuthology.packaging:ref: None 2024-05-15T15:15:47.773 INFO:teuthology.packaging:tag: None 2024-05-15T15:15:47.773 INFO:teuthology.packaging:branch: wip-65480-squid 2024-05-15T15:15:47.773 INFO:teuthology.packaging:sha1: de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:15:47.773 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-65480-squid 2024-05-15T15:15:48.092 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2576.gde5210d5 2024-05-15T15:15:48.094 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-05-15T15:15:48.102 INFO:teuthology.task.internal:no buildpackages task found 2024-05-15T15:15:48.103 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-05-15T15:15:48.223 INFO:teuthology.task.internal:Saving configuration 2024-05-15T15:15:48.242 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-05-15T15:15:48.274 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-15T15:15:48.299 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi043.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-15 15:11:03.052455', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEV/s1jhomw6GouUuRgv82hZgpb9i/dUDfIiPcDZ6e6p9Sm6A++kvvAuToPtje3gSPZk95DuIt1iFCeE+cblstY='} 2024-05-15T15:15:48.325 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi053.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-15 15:11:03.050486', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNW37ucWkq64eR0bVE4G0GhITfumYcKeYmmwMjxzF41yJvjbxq+X+l6WWaq8t8dD58kOevOQU9kV5mow7L3cPZI='} 2024-05-15T15:15:48.325 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-05-15T15:15:48.361 INFO:teuthology.task.internal:roles: ubuntu@smithi043.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-05-15T15:15:48.362 INFO:teuthology.task.internal:roles: ubuntu@smithi053.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-05-15T15:15:48.362 INFO:teuthology.run_tasks:Running task console_log... 2024-05-15T15:15:48.511 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fbb453d30d0>, signals=[15]) 2024-05-15T15:15:48.511 INFO:teuthology.run_tasks:Running task internal.connect... 2024-05-15T15:15:48.523 INFO:teuthology.task.internal:Opening connections... 2024-05-15T15:15:48.523 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi043.front.sepia.ceph.com 2024-05-15T15:15:48.525 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi043.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-15T15:15:48.606 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi053.front.sepia.ceph.com 2024-05-15T15:15:48.607 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-15T15:15:48.682 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-05-15T15:15:48.693 DEBUG:teuthology.orchestra.run.smithi043:> uname -m 2024-05-15T15:15:48.715 INFO:teuthology.orchestra.run.smithi043.stdout:x86_64 2024-05-15T15:15:48.716 DEBUG:teuthology.orchestra.run.smithi043:> cat /etc/os-release 2024-05-15T15:15:48.773 INFO:teuthology.orchestra.run.smithi043.stdout:NAME="CentOS Stream" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:VERSION="9" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:ID="centos" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:ID_LIKE="rhel fedora" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:VERSION_ID="9" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:PLATFORM_ID="platform:el9" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:ANSI_COLOR="0;31" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:LOGO="fedora-logo-icon" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:HOME_URL="https://centos.org/" 2024-05-15T15:15:48.774 INFO:teuthology.orchestra.run.smithi043.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-15T15:15:48.775 INFO:teuthology.orchestra.run.smithi043.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-15T15:15:48.775 INFO:teuthology.orchestra.run.smithi043.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-15T15:15:48.775 INFO:teuthology.lock.ops:Updating smithi043.front.sepia.ceph.com on lock server 2024-05-15T15:15:48.797 DEBUG:teuthology.orchestra.run.smithi053:> uname -m 2024-05-15T15:15:48.816 INFO:teuthology.orchestra.run.smithi053.stdout:x86_64 2024-05-15T15:15:48.817 DEBUG:teuthology.orchestra.run.smithi053:> cat /etc/os-release 2024-05-15T15:15:48.876 INFO:teuthology.orchestra.run.smithi053.stdout:NAME="CentOS Stream" 2024-05-15T15:15:48.876 INFO:teuthology.orchestra.run.smithi053.stdout:VERSION="9" 2024-05-15T15:15:48.876 INFO:teuthology.orchestra.run.smithi053.stdout:ID="centos" 2024-05-15T15:15:48.876 INFO:teuthology.orchestra.run.smithi053.stdout:ID_LIKE="rhel fedora" 2024-05-15T15:15:48.876 INFO:teuthology.orchestra.run.smithi053.stdout:VERSION_ID="9" 2024-05-15T15:15:48.877 INFO:teuthology.orchestra.run.smithi053.stdout:PLATFORM_ID="platform:el9" 2024-05-15T15:15:48.877 INFO:teuthology.orchestra.run.smithi053.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-15T15:15:48.877 INFO:teuthology.orchestra.run.smithi053.stdout:ANSI_COLOR="0;31" 2024-05-15T15:15:48.877 INFO:teuthology.orchestra.run.smithi053.stdout:LOGO="fedora-logo-icon" 2024-05-15T15:15:48.877 INFO:teuthology.orchestra.run.smithi053.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-15T15:15:48.877 INFO:teuthology.orchestra.run.smithi053.stdout:HOME_URL="https://centos.org/" 2024-05-15T15:15:48.877 INFO:teuthology.orchestra.run.smithi053.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-15T15:15:48.877 INFO:teuthology.orchestra.run.smithi053.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-15T15:15:48.877 INFO:teuthology.orchestra.run.smithi053.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-15T15:15:48.877 INFO:teuthology.lock.ops:Updating smithi053.front.sepia.ceph.com on lock server 2024-05-15T15:15:48.904 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-05-15T15:15:48.920 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-05-15T15:15:48.930 INFO:teuthology.task.internal:Checking for old test directory... 2024-05-15T15:15:48.930 DEBUG:teuthology.orchestra.run.smithi043:> test '!' -e /home/ubuntu/cephtest 2024-05-15T15:15:48.933 DEBUG:teuthology.orchestra.run.smithi053:> test '!' -e /home/ubuntu/cephtest 2024-05-15T15:15:48.950 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-05-15T15:15:48.961 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-05-15T15:15:48.961 DEBUG:teuthology.orchestra.run.smithi043:> test -z $(ls -A /var/lib/ceph) 2024-05-15T15:15:48.993 DEBUG:teuthology.orchestra.run.smithi053:> test -z $(ls -A /var/lib/ceph) 2024-05-15T15:15:49.027 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-05-15T15:15:49.130 INFO:teuthology.run_tasks:Running task kernel... 2024-05-15T15:15:49.149 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-05-15T15:15:49.149 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-05-15T15:15:49.149 DEBUG:teuthology.orchestra.run.smithi043:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-15T15:15:49.150 DEBUG:teuthology.orchestra.run.smithi053:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-15T15:15:49.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-15T15:15:49.167 DEBUG:teuthology.orchestra.run.smithi043:> uname -r 2024-05-15T15:15:49.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-15T15:15:49.168 DEBUG:teuthology.orchestra.run.smithi053:> uname -r 2024-05-15T15:15:49.227 INFO:teuthology.orchestra.run.smithi043.stdout:5.14.0-437.el9.x86_64 2024-05-15T15:15:49.227 INFO:teuthology.task.kernel:Running kernel on smithi043: 5.14.0-437.el9.x86_64 2024-05-15T15:15:49.227 DEBUG:teuthology.orchestra.run.smithi043:> sudo yum install -y kernel 2024-05-15T15:15:49.229 INFO:teuthology.orchestra.run.smithi053.stdout:5.14.0-437.el9.x86_64 2024-05-15T15:15:49.230 INFO:teuthology.task.kernel:Running kernel on smithi053: 5.14.0-437.el9.x86_64 2024-05-15T15:15:49.230 DEBUG:teuthology.orchestra.run.smithi053:> sudo yum install -y kernel 2024-05-15T15:15:53.084 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 9 - BaseOS 3.6 MB/s | 8.1 MB 00:02 2024-05-15T15:15:53.219 INFO:teuthology.orchestra.run.smithi043.stdout:CentOS Stream 9 - BaseOS 3.4 MB/s | 8.1 MB 00:02 2024-05-15T15:15:55.012 INFO:teuthology.orchestra.run.smithi043.stdout:CentOS Stream 9 - AppStream 30 MB/s | 19 MB 00:00 2024-05-15T15:15:55.403 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 9 - AppStream 16 MB/s | 19 MB 00:01 2024-05-15T15:15:59.204 INFO:teuthology.orchestra.run.smithi043.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.2 MB 00:00 2024-05-15T15:16:00.127 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 9 - CRB 6.2 MB/s | 6.2 MB 00:01 2024-05-15T15:16:00.822 INFO:teuthology.orchestra.run.smithi043.stdout:CentOS Stream 9 - Extras packages 43 kB/s | 16 kB 00:00 2024-05-15T15:16:01.594 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 9 - Extras packages 62 kB/s | 16 kB 00:00 2024-05-15T15:16:02.088 INFO:teuthology.orchestra.run.smithi043.stdout:Extra Packages for Enterprise Linux 19 MB/s | 21 MB 00:01 2024-05-15T15:16:03.598 INFO:teuthology.orchestra.run.smithi053.stdout:Extra Packages for Enterprise Linux 11 MB/s | 21 MB 00:01 2024-05-15T15:16:07.406 INFO:teuthology.orchestra.run.smithi043.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-05-15T15:16:08.862 INFO:teuthology.orchestra.run.smithi053.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-05-15T15:16:09.487 INFO:teuthology.orchestra.run.smithi043.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-15T15:16:09.488 INFO:teuthology.orchestra.run.smithi043.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-15T15:16:09.588 INFO:teuthology.orchestra.run.smithi043.stdout:Dependencies resolved. 2024-05-15T15:16:09.594 INFO:teuthology.orchestra.run.smithi043.stdout:================================================================================ 2024-05-15T15:16:09.594 INFO:teuthology.orchestra.run.smithi043.stdout: Package Architecture Version Repository Size 2024-05-15T15:16:09.594 INFO:teuthology.orchestra.run.smithi043.stdout:================================================================================ 2024-05-15T15:16:09.594 INFO:teuthology.orchestra.run.smithi043.stdout:Installing: 2024-05-15T15:16:09.594 INFO:teuthology.orchestra.run.smithi043.stdout: kernel x86_64 5.14.0-446.el9 baseos 550 k 2024-05-15T15:16:09.594 INFO:teuthology.orchestra.run.smithi043.stdout:Installing dependencies: 2024-05-15T15:16:09.594 INFO:teuthology.orchestra.run.smithi043.stdout: kernel-core x86_64 5.14.0-446.el9 baseos 15 M 2024-05-15T15:16:09.595 INFO:teuthology.orchestra.run.smithi043.stdout: kernel-modules x86_64 5.14.0-446.el9 baseos 34 M 2024-05-15T15:16:09.595 INFO:teuthology.orchestra.run.smithi043.stdout: kernel-modules-core x86_64 5.14.0-446.el9 baseos 28 M 2024-05-15T15:16:09.595 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:16:09.595 INFO:teuthology.orchestra.run.smithi043.stdout:Transaction Summary 2024-05-15T15:16:09.595 INFO:teuthology.orchestra.run.smithi043.stdout:================================================================================ 2024-05-15T15:16:09.595 INFO:teuthology.orchestra.run.smithi043.stdout:Install 4 Packages 2024-05-15T15:16:09.595 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:16:09.596 INFO:teuthology.orchestra.run.smithi043.stdout:Total download size: 78 M 2024-05-15T15:16:09.596 INFO:teuthology.orchestra.run.smithi043.stdout:Installed size: 123 M 2024-05-15T15:16:09.596 INFO:teuthology.orchestra.run.smithi043.stdout:Downloading Packages: 2024-05-15T15:16:09.891 INFO:teuthology.orchestra.run.smithi043.stdout:(1/4): kernel-5.14.0-446.el9.x86_64.rpm 2.1 MB/s | 550 kB 00:00 2024-05-15T15:16:10.960 INFO:teuthology.orchestra.run.smithi053.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-15T15:16:10.960 INFO:teuthology.orchestra.run.smithi053.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-15T15:16:11.060 INFO:teuthology.orchestra.run.smithi053.stdout:Dependencies resolved. 2024-05-15T15:16:11.064 INFO:teuthology.orchestra.run.smithi053.stdout:================================================================================ 2024-05-15T15:16:11.064 INFO:teuthology.orchestra.run.smithi053.stdout: Package Architecture Version Repository Size 2024-05-15T15:16:11.064 INFO:teuthology.orchestra.run.smithi053.stdout:================================================================================ 2024-05-15T15:16:11.064 INFO:teuthology.orchestra.run.smithi053.stdout:Installing: 2024-05-15T15:16:11.064 INFO:teuthology.orchestra.run.smithi053.stdout: kernel x86_64 5.14.0-446.el9 baseos 550 k 2024-05-15T15:16:11.064 INFO:teuthology.orchestra.run.smithi053.stdout:Installing dependencies: 2024-05-15T15:16:11.064 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-core x86_64 5.14.0-446.el9 baseos 15 M 2024-05-15T15:16:11.065 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-modules x86_64 5.14.0-446.el9 baseos 34 M 2024-05-15T15:16:11.065 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-modules-core x86_64 5.14.0-446.el9 baseos 28 M 2024-05-15T15:16:11.065 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-05-15T15:16:11.065 INFO:teuthology.orchestra.run.smithi053.stdout:Transaction Summary 2024-05-15T15:16:11.065 INFO:teuthology.orchestra.run.smithi053.stdout:================================================================================ 2024-05-15T15:16:11.065 INFO:teuthology.orchestra.run.smithi053.stdout:Install 4 Packages 2024-05-15T15:16:11.065 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-05-15T15:16:11.066 INFO:teuthology.orchestra.run.smithi053.stdout:Total download size: 78 M 2024-05-15T15:16:11.066 INFO:teuthology.orchestra.run.smithi053.stdout:Installed size: 123 M 2024-05-15T15:16:11.066 INFO:teuthology.orchestra.run.smithi053.stdout:Downloading Packages: 2024-05-15T15:16:11.572 INFO:teuthology.orchestra.run.smithi053.stdout:(1/4): kernel-5.14.0-446.el9.x86_64.rpm 2.3 MB/s | 550 kB 00:00 2024-05-15T15:16:12.816 INFO:teuthology.orchestra.run.smithi043.stdout:(2/4): kernel-core-5.14.0-446.el9.x86_64.rpm 4.9 MB/s | 15 MB 00:03 2024-05-15T15:16:13.814 INFO:teuthology.orchestra.run.smithi053.stdout:(2/4): kernel-core-5.14.0-446.el9.x86_64.rpm 6.3 MB/s | 15 MB 00:02 2024-05-15T15:16:14.208 INFO:teuthology.orchestra.run.smithi043.stdout:(3/4): kernel-modules-core-5.14.0-446.el9.x86_6 6.5 MB/s | 28 MB 00:04 2024-05-15T15:16:15.516 INFO:teuthology.orchestra.run.smithi043.stdout:(4/4): kernel-modules-5.14.0-446.el9.x86_64.rpm 5.8 MB/s | 34 MB 00:05 2024-05-15T15:16:15.516 INFO:teuthology.orchestra.run.smithi043.stdout:-------------------------------------------------------------------------------- 2024-05-15T15:16:15.516 INFO:teuthology.orchestra.run.smithi043.stdout:Total 13 MB/s | 78 MB 00:05 2024-05-15T15:16:15.888 INFO:teuthology.orchestra.run.smithi053.stdout:(3/4): kernel-modules-core-5.14.0-446.el9.x86_6 6.5 MB/s | 28 MB 00:04 2024-05-15T15:16:16.230 INFO:teuthology.orchestra.run.smithi043.stdout:Running transaction check 2024-05-15T15:16:16.631 INFO:teuthology.orchestra.run.smithi043.stdout:Transaction check succeeded. 2024-05-15T15:16:16.631 INFO:teuthology.orchestra.run.smithi043.stdout:Running transaction test 2024-05-15T15:16:17.589 INFO:teuthology.orchestra.run.smithi043.stdout:Transaction test succeeded. 2024-05-15T15:16:17.589 INFO:teuthology.orchestra.run.smithi043.stdout:Running transaction 2024-05-15T15:16:17.989 INFO:teuthology.orchestra.run.smithi053.stdout:(4/4): kernel-modules-5.14.0-446.el9.x86_64.rpm 5.1 MB/s | 34 MB 00:06 2024-05-15T15:16:17.989 INFO:teuthology.orchestra.run.smithi053.stdout:-------------------------------------------------------------------------------- 2024-05-15T15:16:17.989 INFO:teuthology.orchestra.run.smithi053.stdout:Total 11 MB/s | 78 MB 00:06 2024-05-15T15:16:18.701 INFO:teuthology.orchestra.run.smithi053.stdout:Running transaction check 2024-05-15T15:16:19.110 INFO:teuthology.orchestra.run.smithi053.stdout:Transaction check succeeded. 2024-05-15T15:16:19.110 INFO:teuthology.orchestra.run.smithi053.stdout:Running transaction test 2024-05-15T15:16:20.074 INFO:teuthology.orchestra.run.smithi053.stdout:Transaction test succeeded. 2024-05-15T15:16:20.074 INFO:teuthology.orchestra.run.smithi053.stdout:Running transaction 2024-05-15T15:16:20.122 INFO:teuthology.orchestra.run.smithi043.stdout: Preparing : 1/1 2024-05-15T15:16:20.746 INFO:teuthology.orchestra.run.smithi043.stdout: Installing : kernel-modules-core-5.14.0-446.el9.x86_64 1/4 2024-05-15T15:16:20.825 INFO:teuthology.orchestra.run.smithi043.stdout: Installing : kernel-core-5.14.0-446.el9.x86_64 2/4 2024-05-15T15:16:22.614 INFO:teuthology.orchestra.run.smithi053.stdout: Preparing : 1/1 2024-05-15T15:16:23.252 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-modules-core-5.14.0-446.el9.x86_64 1/4 2024-05-15T15:16:23.330 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-core-5.14.0-446.el9.x86_64 2/4 2024-05-15T15:16:23.624 INFO:teuthology.orchestra.run.smithi043.stdout: Running scriptlet: kernel-core-5.14.0-446.el9.x86_64 2/4 2024-05-15T15:16:23.755 INFO:teuthology.orchestra.run.smithi043.stdout: Installing : kernel-modules-5.14.0-446.el9.x86_64 3/4 2024-05-15T15:16:26.127 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-core-5.14.0-446.el9.x86_64 2/4 2024-05-15T15:16:26.267 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-modules-5.14.0-446.el9.x86_64 3/4 2024-05-15T15:16:28.959 INFO:teuthology.orchestra.run.smithi043.stdout: Running scriptlet: kernel-modules-5.14.0-446.el9.x86_64 3/4 2024-05-15T15:16:29.009 INFO:teuthology.orchestra.run.smithi043.stdout: Installing : kernel-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:16:31.345 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-modules-5.14.0-446.el9.x86_64 3/4 2024-05-15T15:16:31.395 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:16:34.109 INFO:teuthology.orchestra.run.smithi043.stdout: Running scriptlet: kernel-modules-core-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:16:36.480 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-modules-core-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:17:05.011 INFO:teuthology.orchestra.run.smithi043.stdout: Running scriptlet: kernel-core-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:17:05.051 INFO:teuthology.orchestra.run.smithi043.stdout: Running scriptlet: kernel-modules-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:17:05.350 INFO:teuthology.orchestra.run.smithi043.stdout: Running scriptlet: kernel-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:17:05.350 INFO:teuthology.orchestra.run.smithi043.stdout: Verifying : kernel-5.14.0-446.el9.x86_64 1/4 2024-05-15T15:17:05.351 INFO:teuthology.orchestra.run.smithi043.stdout: Verifying : kernel-core-5.14.0-446.el9.x86_64 2/4 2024-05-15T15:17:05.351 INFO:teuthology.orchestra.run.smithi043.stdout: Verifying : kernel-modules-5.14.0-446.el9.x86_64 3/4 2024-05-15T15:17:06.146 INFO:teuthology.orchestra.run.smithi043.stdout: Verifying : kernel-modules-core-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:17:06.146 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:17:06.146 INFO:teuthology.orchestra.run.smithi043.stdout:Installed: 2024-05-15T15:17:06.146 INFO:teuthology.orchestra.run.smithi043.stdout: kernel-5.14.0-446.el9.x86_64 2024-05-15T15:17:06.146 INFO:teuthology.orchestra.run.smithi043.stdout: kernel-core-5.14.0-446.el9.x86_64 2024-05-15T15:17:06.146 INFO:teuthology.orchestra.run.smithi043.stdout: kernel-modules-5.14.0-446.el9.x86_64 2024-05-15T15:17:06.146 INFO:teuthology.orchestra.run.smithi043.stdout: kernel-modules-core-5.14.0-446.el9.x86_64 2024-05-15T15:17:06.147 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:17:06.147 INFO:teuthology.orchestra.run.smithi043.stdout:Complete! 2024-05-15T15:17:06.483 DEBUG:teuthology.orchestra.run.smithi043:> rpm -q kernel | sort -rV | head -n 1 2024-05-15T15:17:06.563 INFO:teuthology.orchestra.run.smithi043.stdout:kernel-5.14.0-446.el9.x86_64 2024-05-15T15:17:06.563 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-446.el9.x86_64 2024-05-15T15:17:06.563 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-446.el9.x86_64 2024-05-15T15:17:06.564 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-15T15:17:06.564 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-15T15:17:06.564 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-05-15T15:17:06.564 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi043.front.sepia.ceph.com, path=None, version=distro) 2024-05-15T15:17:06.564 DEBUG:teuthology.orchestra.run.smithi043:> rpm -q kernel | sort -rV | head -n 1 2024-05-15T15:17:06.615 INFO:teuthology.orchestra.run.smithi043.stdout:kernel-5.14.0-446.el9.x86_64 2024-05-15T15:17:06.616 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-446.el9.x86_64 2024-05-15T15:17:06.616 DEBUG:teuthology.orchestra.run.smithi043:> sudo rpm -qi grub2-tools 2024-05-15T15:17:06.650 INFO:teuthology.orchestra.run.smithi043.stdout:Name : grub2-tools 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Epoch : 1 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Version : 2.06 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Release : 68.el9 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Architecture: x86_64 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Group : Unspecified 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Size : 8264254 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:License : GPLv3+ 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Packager : builder@centos.org 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:Vendor : CentOS 2024-05-15T15:17:06.651 INFO:teuthology.orchestra.run.smithi043.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-15T15:17:06.652 INFO:teuthology.orchestra.run.smithi043.stdout:Summary : Support tools for GRUB. 2024-05-15T15:17:06.652 INFO:teuthology.orchestra.run.smithi043.stdout:Description : 2024-05-15T15:17:06.652 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:17:06.652 INFO:teuthology.orchestra.run.smithi043.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-15T15:17:06.652 INFO:teuthology.orchestra.run.smithi043.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-15T15:17:06.652 INFO:teuthology.orchestra.run.smithi043.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-15T15:17:06.652 INFO:teuthology.orchestra.run.smithi043.stdout:hardware devices. 2024-05-15T15:17:06.652 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:17:06.652 INFO:teuthology.orchestra.run.smithi043.stdout:This subpackage provides tools for support of all platforms. 2024-05-15T15:17:06.653 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-15T15:17:06.653 INFO:teuthology.task.kernel:Updating grub on smithi043 to boot 5.14.0-446.el9.x86_64 2024-05-15T15:17:06.653 DEBUG:teuthology.orchestra.run.smithi043:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-15T15:17:07.216 INFO:teuthology.orchestra.run.smithi043.stderr:Generating grub configuration file ... 2024-05-15T15:17:07.266 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-core-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:17:07.306 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-modules-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:17:07.649 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:17:07.650 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-5.14.0-446.el9.x86_64 1/4 2024-05-15T15:17:07.650 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-core-5.14.0-446.el9.x86_64 2/4 2024-05-15T15:17:07.650 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-modules-5.14.0-446.el9.x86_64 3/4 2024-05-15T15:17:08.433 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-modules-core-5.14.0-446.el9.x86_64 4/4 2024-05-15T15:17:08.434 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-05-15T15:17:08.434 INFO:teuthology.orchestra.run.smithi053.stdout:Installed: 2024-05-15T15:17:08.434 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-5.14.0-446.el9.x86_64 2024-05-15T15:17:08.434 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-core-5.14.0-446.el9.x86_64 2024-05-15T15:17:08.434 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-modules-5.14.0-446.el9.x86_64 2024-05-15T15:17:08.434 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-modules-core-5.14.0-446.el9.x86_64 2024-05-15T15:17:08.434 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-05-15T15:17:08.434 INFO:teuthology.orchestra.run.smithi053.stdout:Complete! 2024-05-15T15:17:08.586 INFO:teuthology.orchestra.run.smithi043.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-15T15:17:08.604 INFO:teuthology.orchestra.run.smithi043.stderr:done 2024-05-15T15:17:08.606 DEBUG:teuthology.orchestra.run.smithi043:> mktemp 2024-05-15T15:17:08.621 INFO:teuthology.orchestra.run.smithi043.stdout:/tmp/tmp.qrenwgnXi7 2024-05-15T15:17:08.621 DEBUG:teuthology.orchestra.run.smithi043:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.qrenwgnXi7 2024-05-15T15:17:08.686 DEBUG:teuthology.orchestra.run.smithi043:> sudo chmod 0666 /tmp/tmp.qrenwgnXi7 2024-05-15T15:17:08.803 DEBUG:teuthology.orchestra.run.smithi053:> rpm -q kernel | sort -rV | head -n 1 2024-05-15T15:17:08.824 DEBUG:teuthology.orchestra.remote:smithi043:/tmp/tmp.qrenwgnXi7 is 6KB 2024-05-15T15:17:08.874 DEBUG:teuthology.orchestra.run.smithi043:> rm -fr /tmp/tmp.qrenwgnXi7 2024-05-15T15:17:08.881 INFO:teuthology.orchestra.run.smithi053.stdout:kernel-5.14.0-446.el9.x86_64 2024-05-15T15:17:08.881 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-446.el9.x86_64 2024-05-15T15:17:08.881 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-446.el9.x86_64 2024-05-15T15:17:08.881 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-15T15:17:08.881 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-15T15:17:08.881 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-05-15T15:17:08.881 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi053.front.sepia.ceph.com, path=None, version=distro) 2024-05-15T15:17:08.881 DEBUG:teuthology.orchestra.run.smithi053:> rpm -q kernel | sort -rV | head -n 1 2024-05-15T15:17:08.889 DEBUG:teuthology.orchestra.run.smithi043:> sudo /bin/ls /boot/loader/entries || true 2024-05-15T15:17:08.932 INFO:teuthology.orchestra.run.smithi053.stdout:kernel-5.14.0-446.el9.x86_64 2024-05-15T15:17:08.933 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-446.el9.x86_64 2024-05-15T15:17:08.933 DEBUG:teuthology.orchestra.run.smithi053:> sudo rpm -qi grub2-tools 2024-05-15T15:17:08.951 INFO:teuthology.orchestra.run.smithi043.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-15T15:17:08.951 INFO:teuthology.orchestra.run.smithi043.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-15T15:17:08.951 INFO:teuthology.orchestra.run.smithi043.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-15T15:17:08.951 INFO:teuthology.orchestra.run.smithi043.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-446.el9.x86_64.conf 2024-05-15T15:17:08.952 DEBUG:teuthology.orchestra.run.smithi043:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-446.el9.x86_64 2024-05-15T15:17:08.964 INFO:teuthology.orchestra.run.smithi053.stdout:Name : grub2-tools 2024-05-15T15:17:08.964 INFO:teuthology.orchestra.run.smithi053.stdout:Epoch : 1 2024-05-15T15:17:08.964 INFO:teuthology.orchestra.run.smithi053.stdout:Version : 2.06 2024-05-15T15:17:08.964 INFO:teuthology.orchestra.run.smithi053.stdout:Release : 68.el9 2024-05-15T15:17:08.964 INFO:teuthology.orchestra.run.smithi053.stdout:Architecture: x86_64 2024-05-15T15:17:08.964 INFO:teuthology.orchestra.run.smithi053.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:Group : Unspecified 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:Size : 8264254 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:License : GPLv3+ 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:Packager : builder@centos.org 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:Vendor : CentOS 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:Summary : Support tools for GRUB. 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:Description : 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-05-15T15:17:08.965 INFO:teuthology.orchestra.run.smithi053.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-15T15:17:08.966 INFO:teuthology.orchestra.run.smithi053.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-15T15:17:08.966 INFO:teuthology.orchestra.run.smithi053.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-15T15:17:08.966 INFO:teuthology.orchestra.run.smithi053.stdout:hardware devices. 2024-05-15T15:17:08.966 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-05-15T15:17:08.966 INFO:teuthology.orchestra.run.smithi053.stdout:This subpackage provides tools for support of all platforms. 2024-05-15T15:17:08.967 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-15T15:17:08.967 INFO:teuthology.task.kernel:Updating grub on smithi053 to boot 5.14.0-446.el9.x86_64 2024-05-15T15:17:08.967 DEBUG:teuthology.orchestra.run.smithi053:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-15T15:17:09.112 DEBUG:teuthology.orchestra.run.smithi043:> sudo shutdown -r now 2024-05-15T15:17:09.630 INFO:teuthology.orchestra.run.smithi053.stderr:Generating grub configuration file ... 2024-05-15T15:17:11.000 INFO:teuthology.orchestra.run.smithi053.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-15T15:17:11.044 INFO:teuthology.orchestra.run.smithi053.stderr:done 2024-05-15T15:17:11.046 DEBUG:teuthology.orchestra.run.smithi053:> mktemp 2024-05-15T15:17:11.061 INFO:teuthology.orchestra.run.smithi053.stdout:/tmp/tmp.6nLqe5KBc4 2024-05-15T15:17:11.062 DEBUG:teuthology.orchestra.run.smithi053:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.6nLqe5KBc4 2024-05-15T15:17:11.137 DEBUG:teuthology.orchestra.run.smithi053:> sudo chmod 0666 /tmp/tmp.6nLqe5KBc4 2024-05-15T15:17:11.271 DEBUG:teuthology.orchestra.remote:smithi053:/tmp/tmp.6nLqe5KBc4 is 6KB 2024-05-15T15:17:11.319 DEBUG:teuthology.orchestra.run.smithi053:> rm -fr /tmp/tmp.6nLqe5KBc4 2024-05-15T15:17:11.334 DEBUG:teuthology.orchestra.run.smithi053:> sudo /bin/ls /boot/loader/entries || true 2024-05-15T15:17:11.396 INFO:teuthology.orchestra.run.smithi053.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-15T15:17:11.396 INFO:teuthology.orchestra.run.smithi053.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-15T15:17:11.397 INFO:teuthology.orchestra.run.smithi053.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-15T15:17:11.397 INFO:teuthology.orchestra.run.smithi053.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-446.el9.x86_64.conf 2024-05-15T15:17:11.398 DEBUG:teuthology.orchestra.run.smithi053:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-446.el9.x86_64 2024-05-15T15:17:11.533 DEBUG:teuthology.orchestra.run.smithi053:> sudo shutdown -r now 2024-05-15T15:17:39.142 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-05-15T15:17:39.143 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi043.front.sepia.ceph.com' 2024-05-15T15:17:39.145 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi043.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-15T15:17:41.538 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-05-15T15:17:41.539 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-05-15T15:17:41.539 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-15T15:17:57.667 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.43 2024-05-15T15:17:59.970 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.53 2024-05-15T15:18:06.674 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi043.front.sepia.ceph.com' 2024-05-15T15:18:06.675 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi043.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-15T15:18:08.972 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-05-15T15:18:08.973 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-15T15:18:27.359 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.53 2024-05-15T15:18:39.370 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-05-15T15:18:39.372 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-15T15:18:39.779 DEBUG:teuthology.orchestra.run.smithi053:> true 2024-05-15T15:18:40.198 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-05-15T15:18:40.198 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-446.el9.x86_64"... 2024-05-15T15:18:40.198 DEBUG:teuthology.orchestra.run.smithi053:> uname -r 2024-05-15T15:18:40.214 INFO:teuthology.orchestra.run.smithi053.stdout:5.14.0-446.el9.x86_64 2024-05-15T15:18:40.214 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-446.el9.x86_64 vs 5.14.0-446.el9.x86_64 2024-05-15T15:18:40.214 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-15T15:18:40.214 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-15T15:18:41.215 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-05-15T15:18:41.215 DEBUG:teuthology.orchestra.run.smithi053:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-15T15:18:41.288 INFO:teuthology.orchestra.run.smithi053.stdout:ttyS1 2024-05-15T15:18:41.329 DEBUG:teuthology.parallel:result is None 2024-05-15T15:19:06.677 DEBUG:teuthology.orchestra.remote:timed out 2024-05-15T15:19:18.681 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi043.front.sepia.ceph.com' 2024-05-15T15:19:18.683 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi043.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-15T15:19:19.097 DEBUG:teuthology.orchestra.run.smithi043:> true 2024-05-15T15:19:19.490 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi043.front.sepia.ceph.com' 2024-05-15T15:19:19.490 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-446.el9.x86_64"... 2024-05-15T15:19:19.491 DEBUG:teuthology.orchestra.run.smithi043:> uname -r 2024-05-15T15:19:19.507 INFO:teuthology.orchestra.run.smithi043.stdout:5.14.0-446.el9.x86_64 2024-05-15T15:19:19.508 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-446.el9.x86_64 vs 5.14.0-446.el9.x86_64 2024-05-15T15:19:19.508 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-15T15:19:19.508 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-15T15:19:20.509 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-05-15T15:19:20.509 DEBUG:teuthology.orchestra.run.smithi043:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-15T15:19:20.575 INFO:teuthology.orchestra.run.smithi043.stdout:ttyS1 2024-05-15T15:19:20.608 DEBUG:teuthology.parallel:result is None 2024-05-15T15:19:20.608 INFO:teuthology.run_tasks:Running task internal.base... 2024-05-15T15:19:20.616 INFO:teuthology.task.internal:Creating test directory... 2024-05-15T15:19:20.616 DEBUG:teuthology.orchestra.run.smithi043:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-15T15:19:20.618 DEBUG:teuthology.orchestra.run.smithi053:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-15T15:19:20.640 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-05-15T15:19:20.646 INFO:teuthology.run_tasks:Running task internal.archive... 2024-05-15T15:19:20.657 INFO:teuthology.task.internal:Creating archive directory... 2024-05-15T15:19:20.658 DEBUG:teuthology.orchestra.run.smithi043:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-15T15:19:20.678 DEBUG:teuthology.orchestra.run.smithi053:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-15T15:19:20.717 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-05-15T15:19:20.724 INFO:teuthology.task.internal:Enabling coredump saving... 2024-05-15T15:19:20.725 DEBUG:teuthology.orchestra.run.smithi043:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-15T15:19:20.753 DEBUG:teuthology.orchestra.run.smithi053:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-15T15:19:20.788 INFO:teuthology.orchestra.run.smithi053.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-15T15:19:20.790 INFO:teuthology.orchestra.run.smithi043.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-15T15:19:20.800 INFO:teuthology.orchestra.run.smithi053.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-15T15:19:20.802 INFO:teuthology.orchestra.run.smithi043.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-15T15:19:20.804 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-05-15T15:19:20.811 INFO:teuthology.task.internal:Configuring sudo... 2024-05-15T15:19:20.812 DEBUG:teuthology.orchestra.run.smithi043:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-15T15:19:20.847 DEBUG:teuthology.orchestra.run.smithi053:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-15T15:19:20.876 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-05-15T15:19:20.886 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-05-15T15:19:20.886 DEBUG:teuthology.orchestra.run.smithi043:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-15T15:19:20.917 DEBUG:teuthology.orchestra.run.smithi053:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-15T15:19:20.935 DEBUG:teuthology.orchestra.run.smithi043:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-15T15:19:21.021 DEBUG:teuthology.orchestra.run.smithi043:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-15T15:19:21.068 DEBUG:teuthology.orchestra.run.smithi043:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-15T15:19:21.154 DEBUG:teuthology.orchestra.run.smithi043:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-15T15:19:21.197 DEBUG:teuthology.orchestra.run.smithi043:> set -ex 2024-05-15T15:19:21.197 DEBUG:teuthology.orchestra.run.smithi043:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-15T15:19:21.265 DEBUG:teuthology.orchestra.run.smithi053:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-15T15:19:21.311 DEBUG:teuthology.orchestra.run.smithi053:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-15T15:19:21.367 DEBUG:teuthology.orchestra.run.smithi053:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-15T15:19:21.453 DEBUG:teuthology.orchestra.run.smithi053:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-15T15:19:21.497 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-05-15T15:19:21.497 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-15T15:19:21.565 DEBUG:teuthology.orchestra.run.smithi043:> sudo service rsyslog restart 2024-05-15T15:19:21.567 DEBUG:teuthology.orchestra.run.smithi053:> sudo service rsyslog restart 2024-05-15T15:19:21.610 INFO:teuthology.orchestra.run.smithi043.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-15T15:19:21.660 INFO:teuthology.orchestra.run.smithi053.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-15T15:19:22.123 INFO:teuthology.run_tasks:Running task internal.timer... 2024-05-15T15:19:22.131 INFO:teuthology.task.internal:Starting timer... 2024-05-15T15:19:22.131 INFO:teuthology.run_tasks:Running task pcp... 2024-05-15T15:19:22.141 INFO:teuthology.run_tasks:Running task selinux... 2024-05-15T15:19:22.149 DEBUG:teuthology.orchestra.run.smithi043:> sudo service auditd rotate 2024-05-15T15:19:22.220 INFO:teuthology.orchestra.run.smithi043.stdout:Rotating logs: 2024-05-15T15:19:22.221 DEBUG:teuthology.orchestra.run.smithi053:> sudo service auditd rotate 2024-05-15T15:19:22.314 INFO:teuthology.orchestra.run.smithi053.stdout:Rotating logs: 2024-05-15T15:19:22.316 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-05-15T15:19:22.316 DEBUG:teuthology.orchestra.run.smithi043:> /usr/sbin/getenforce 2024-05-15T15:19:22.344 INFO:teuthology.orchestra.run.smithi043.stdout:Permissive 2024-05-15T15:19:22.345 DEBUG:teuthology.orchestra.run.smithi053:> /usr/sbin/getenforce 2024-05-15T15:19:22.376 INFO:teuthology.orchestra.run.smithi053.stdout:Permissive 2024-05-15T15:19:22.376 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi043.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi053.front.sepia.ceph.com': 'permissive'} 2024-05-15T15:19:22.377 DEBUG:teuthology.orchestra.run.smithi043:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-15T15:19:22.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-15T15:19:22.416 DEBUG:teuthology.orchestra.run.smithi053:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-15T15:19:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-15T15:19:22.446 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-05-15T15:19:22.447 DEBUG:teuthology.orchestra.run.smithi043:> sudo /usr/sbin/setenforce permissive 2024-05-15T15:19:22.483 DEBUG:teuthology.orchestra.run.smithi053:> sudo /usr/sbin/setenforce permissive 2024-05-15T15:19:22.515 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-05-15T15:19:22.525 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-05-15T15:19:22.545 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-05-15T15:19:22.546 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi043.front.sepia.ceph.com,smithi053.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-05-15T15:23:51.704 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi043.front.sepia.ceph.com'), Remote(name='ubuntu@smithi053.front.sepia.ceph.com')] 2024-05-15T15:23:51.705 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi043.front.sepia.ceph.com' 2024-05-15T15:23:51.707 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi043.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-15T15:23:51.787 DEBUG:teuthology.orchestra.run.smithi043:> true 2024-05-15T15:23:51.867 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi043.front.sepia.ceph.com' 2024-05-15T15:23:51.867 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-05-15T15:23:51.867 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-15T15:23:51.956 DEBUG:teuthology.orchestra.run.smithi053:> true 2024-05-15T15:23:52.044 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-05-15T15:23:52.045 INFO:teuthology.run_tasks:Running task clock... 2024-05-15T15:23:52.060 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-05-15T15:23:52.060 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-15T15:23:52.061 DEBUG:teuthology.orchestra.run.smithi043:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-15T15:23:52.063 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-15T15:23:52.064 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-15T15:23:52.093 INFO:teuthology.orchestra.run.smithi043.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-15T15:23:52.107 INFO:teuthology.orchestra.run.smithi043.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-15T15:23:52.116 INFO:teuthology.orchestra.run.smithi053.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-15T15:23:52.132 INFO:teuthology.orchestra.run.smithi043.stderr:sudo: ntpd: command not found 2024-05-15T15:23:52.133 INFO:teuthology.orchestra.run.smithi053.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-15T15:23:52.144 INFO:teuthology.orchestra.run.smithi043.stdout:506 Cannot talk to daemon 2024-05-15T15:23:52.159 INFO:teuthology.orchestra.run.smithi043.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-15T15:23:52.159 INFO:teuthology.orchestra.run.smithi053.stderr:sudo: ntpd: command not found 2024-05-15T15:23:52.171 INFO:teuthology.orchestra.run.smithi053.stdout:506 Cannot talk to daemon 2024-05-15T15:23:52.173 INFO:teuthology.orchestra.run.smithi043.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-15T15:23:52.186 INFO:teuthology.orchestra.run.smithi053.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-15T15:23:52.201 INFO:teuthology.orchestra.run.smithi053.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-15T15:23:52.236 INFO:teuthology.orchestra.run.smithi043.stderr:bash: line 1: ntpq: command not found 2024-05-15T15:23:52.240 INFO:teuthology.orchestra.run.smithi043.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-15T15:23:52.240 INFO:teuthology.orchestra.run.smithi043.stdout:=============================================================================== 2024-05-15T15:23:52.240 INFO:teuthology.orchestra.run.smithi043.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-15T15:23:52.240 INFO:teuthology.orchestra.run.smithi043.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-15T15:23:52.240 INFO:teuthology.orchestra.run.smithi043.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-15T15:23:52.240 INFO:teuthology.orchestra.run.smithi043.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-15T15:23:52.261 INFO:teuthology.orchestra.run.smithi053.stderr:bash: line 1: ntpq: command not found 2024-05-15T15:23:52.264 INFO:teuthology.orchestra.run.smithi053.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-15T15:23:52.264 INFO:teuthology.orchestra.run.smithi053.stdout:=============================================================================== 2024-05-15T15:23:52.265 INFO:teuthology.orchestra.run.smithi053.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-15T15:23:52.265 INFO:teuthology.orchestra.run.smithi053.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-15T15:23:52.265 INFO:teuthology.orchestra.run.smithi053.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-15T15:23:52.265 INFO:teuthology.orchestra.run.smithi053.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-15T15:23:52.266 INFO:teuthology.run_tasks:Running task cephadm... 2024-05-15T15:23:52.389 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': 'de5210d58560a4684fdb5b238ef3e6e0285a1de8'} 2024-05-15T15:23:52.389 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-05-15T15:23:52.389 INFO:tasks.cephadm:Cluster fsid is 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:23:52.389 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-05-15T15:23:52.390 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.43', 'mon.c': '[v2:172.21.15.43:3301,v1:172.21.15.43:6790]', 'mon.b': '172.21.15.53'} 2024-05-15T15:23:52.390 INFO:tasks.cephadm:First mon is mon.a on smithi043 2024-05-15T15:23:52.390 INFO:tasks.cephadm:First mgr is y 2024-05-15T15:23:52.390 INFO:tasks.cephadm:Normalizing hostnames... 2024-05-15T15:23:52.390 DEBUG:teuthology.orchestra.run.smithi043:> sudo hostname $(hostname -s) 2024-05-15T15:23:52.417 DEBUG:teuthology.orchestra.run.smithi053:> sudo hostname $(hostname -s) 2024-05-15T15:23:52.445 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-05-15T15:23:52.445 DEBUG:teuthology.orchestra.run.smithi043:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-15T15:23:52.763 INFO:teuthology.orchestra.run.smithi043.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 15 15:23 /home/ubuntu/cephtest/cephadm 2024-05-15T15:23:52.763 DEBUG:teuthology.orchestra.run.smithi053:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-15T15:23:53.129 INFO:teuthology.orchestra.run.smithi053.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 15 15:23 /home/ubuntu/cephtest/cephadm 2024-05-15T15:23:53.129 DEBUG:teuthology.orchestra.run.smithi043:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-15T15:23:53.147 DEBUG:teuthology.orchestra.run.smithi053:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-15T15:23:53.176 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-05-15T15:23:53.176 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-15T15:23:53.190 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-15T15:23:54.117 INFO:teuthology.orchestra.run.smithi053.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-15T15:23:54.218 INFO:teuthology.orchestra.run.smithi043.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-15T15:24:16.498 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:24:16.498 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-15T15:24:16.498 INFO:teuthology.orchestra.run.smithi043.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-15T15:24:16.498 INFO:teuthology.orchestra.run.smithi043.stdout: "repo_digests": [ 2024-05-15T15:24:16.498 INFO:teuthology.orchestra.run.smithi043.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-15T15:24:16.498 INFO:teuthology.orchestra.run.smithi043.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-15T15:24:16.498 INFO:teuthology.orchestra.run.smithi043.stdout: ] 2024-05-15T15:24:16.498 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:24:19.674 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-05-15T15:24:19.674 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-15T15:24:19.675 INFO:teuthology.orchestra.run.smithi053.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-15T15:24:19.675 INFO:teuthology.orchestra.run.smithi053.stdout: "repo_digests": [ 2024-05-15T15:24:19.675 INFO:teuthology.orchestra.run.smithi053.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-15T15:24:19.675 INFO:teuthology.orchestra.run.smithi053.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-15T15:24:19.675 INFO:teuthology.orchestra.run.smithi053.stdout: ] 2024-05-15T15:24:19.675 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-05-15T15:24:19.694 DEBUG:teuthology.orchestra.run.smithi043:> sudo mkdir -p /etc/ceph 2024-05-15T15:24:19.730 DEBUG:teuthology.orchestra.run.smithi053:> sudo mkdir -p /etc/ceph 2024-05-15T15:24:19.760 DEBUG:teuthology.orchestra.run.smithi043:> sudo chmod 777 /etc/ceph 2024-05-15T15:24:19.801 DEBUG:teuthology.orchestra.run.smithi053:> sudo chmod 777 /etc/ceph 2024-05-15T15:24:19.826 INFO:tasks.cephadm:Writing seed config... 2024-05-15T15:24:19.828 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-05-15T15:24:19.828 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-05-15T15:24:19.828 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-05-15T15:24:19.828 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-05-15T15:24:19.828 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-05-15T15:24:19.828 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-05-15T15:24:19.828 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-05-15T15:24:19.828 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-05-15T15:24:19.828 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-05-15T15:24:19.829 DEBUG:teuthology.orchestra.run.smithi043:> set -ex 2024-05-15T15:24:19.829 DEBUG:teuthology.orchestra.run.smithi043:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-05-15T15:24:19.860 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 22d82864-12cf-11ef-bc98-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-05-15T15:24:19.861 DEBUG:teuthology.orchestra.run.smithi043:mon.a> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.a.service 2024-05-15T15:24:19.903 DEBUG:teuthology.orchestra.run.smithi043:mgr.y> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.y.service 2024-05-15T15:24:19.946 INFO:tasks.cephadm:Bootstrapping... 2024-05-15T15:24:19.946 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 22d82864-12cf-11ef-bc98-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.43 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-05-15T15:24:20.152 INFO:teuthology.orchestra.run.smithi043.stderr:-------------------------------------------------------------------------------- 2024-05-15T15:24:20.152 INFO:teuthology.orchestra.run.smithi043.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '22d82864-12cf-11ef-bc98-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.43', '--skip-admin-label'] 2024-05-15T15:24:20.177 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: 5.0.2 2024-05-15T15:24:20.180 INFO:teuthology.orchestra.run.smithi043.stderr:Verifying podman|docker is present... 2024-05-15T15:24:20.202 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: 5.0.2 2024-05-15T15:24:20.205 INFO:teuthology.orchestra.run.smithi043.stderr:Verifying lvm2 is present... 2024-05-15T15:24:20.205 INFO:teuthology.orchestra.run.smithi043.stderr:Verifying time synchronization is in place... 2024-05-15T15:24:20.211 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-15T15:24:20.218 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: inactive 2024-05-15T15:24:20.225 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: enabled 2024-05-15T15:24:20.231 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: active 2024-05-15T15:24:20.232 INFO:teuthology.orchestra.run.smithi043.stderr:Unit chronyd.service is enabled and running 2024-05-15T15:24:20.232 INFO:teuthology.orchestra.run.smithi043.stderr:Repeating the final host check... 2024-05-15T15:24:20.253 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: 5.0.2 2024-05-15T15:24:20.256 INFO:teuthology.orchestra.run.smithi043.stderr:podman (/bin/podman) version 5.0.2 is present 2024-05-15T15:24:20.256 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl is present 2024-05-15T15:24:20.256 INFO:teuthology.orchestra.run.smithi043.stderr:lvcreate is present 2024-05-15T15:24:20.263 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-15T15:24:20.269 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: inactive 2024-05-15T15:24:20.276 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: enabled 2024-05-15T15:24:20.283 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: active 2024-05-15T15:24:20.284 INFO:teuthology.orchestra.run.smithi043.stderr:Unit chronyd.service is enabled and running 2024-05-15T15:24:20.284 INFO:teuthology.orchestra.run.smithi043.stderr:Host looks OK 2024-05-15T15:24:20.284 INFO:teuthology.orchestra.run.smithi043.stderr:Cluster fsid: 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:24:20.284 INFO:teuthology.orchestra.run.smithi043.stderr:Acquiring lock 140225533731840 on /run/cephadm/22d82864-12cf-11ef-bc98-c7b262605968.lock 2024-05-15T15:24:20.284 INFO:teuthology.orchestra.run.smithi043.stderr:Lock 140225533731840 acquired on /run/cephadm/22d82864-12cf-11ef-bc98-c7b262605968.lock 2024-05-15T15:24:20.284 INFO:teuthology.orchestra.run.smithi043.stderr:Verifying IP 172.21.15.43 port 3300 ... 2024-05-15T15:24:20.285 INFO:teuthology.orchestra.run.smithi043.stderr:Verifying IP 172.21.15.43 port 6789 ... 2024-05-15T15:24:20.285 INFO:teuthology.orchestra.run.smithi043.stderr:Base mon IP is 172.21.15.43, final addrv is [v2:172.21.15.43:3300,v1:172.21.15.43:6789] 2024-05-15T15:24:20.287 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.43 metric 100 2024-05-15T15:24:20.287 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.43 metric 100 2024-05-15T15:24:20.290 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-05-15T15:24:20.290 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-05-15T15:24:20.290 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-05-15T15:24:20.292 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-05-15T15:24:20.293 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-05-15T15:24:20.293 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-15T15:24:20.293 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-05-15T15:24:20.293 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:1358/64 scope link noprefixroute 2024-05-15T15:24:20.293 INFO:teuthology.orchestra.run.smithi043.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-15T15:24:20.293 INFO:teuthology.orchestra.run.smithi043.stderr:Mon IP `172.21.15.43` is in CIDR network `172.21.0.0/20` 2024-05-15T15:24:20.294 INFO:teuthology.orchestra.run.smithi043.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-05-15T15:24:20.294 INFO:teuthology.orchestra.run.smithi043.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-15T15:24:20.399 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-05-15T15:24:20.839 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: Getting image source signatures 2024-05-15T15:24:20.839 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-05-15T15:24:20.840 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-05-15T15:24:20.840 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-05-15T15:24:20.840 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-05-15T15:24:20.840 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-05-15T15:24:20.840 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-15T15:24:20.965 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: Writing manifest to image destination 2024-05-15T15:24:21.101 INFO:teuthology.orchestra.run.smithi043.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-15T15:24:21.548 INFO:teuthology.orchestra.run.smithi043.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-15T15:24:22.026 INFO:teuthology.orchestra.run.smithi043.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-15T15:24:22.026 INFO:teuthology.orchestra.run.smithi043.stderr:Extracting ceph user uid/gid from container image... 2024-05-15T15:24:22.442 INFO:teuthology.orchestra.run.smithi043.stderr:stat: 167 167 2024-05-15T15:24:22.865 INFO:teuthology.orchestra.run.smithi043.stderr:Creating initial keys... 2024-05-15T15:24:23.306 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph-authtool: AQCn00RmXgrJCxAAiB+mw2e6b0Img9FB4fs1+Q== 2024-05-15T15:24:24.094 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph-authtool: AQCo00RmXlEYARAAT8hYsJ/ahgauet3cLiOxlw== 2024-05-15T15:24:24.992 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph-authtool: AQCo00Rm8Y+eOBAASNWTj/K0d5US/LyCxGj++A== 2024-05-15T15:24:25.466 INFO:teuthology.orchestra.run.smithi043.stderr:Creating initial monmap... 2024-05-15T15:24:25.881 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-15T15:24:25.881 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-05-15T15:24:25.881 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:24:25.881 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-15T15:24:26.397 INFO:teuthology.orchestra.run.smithi043.stderr:monmaptool for a [v2:172.21.15.43:3300,v1:172.21.15.43:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-15T15:24:26.397 INFO:teuthology.orchestra.run.smithi043.stderr:setting min_mon_release = octopus 2024-05-15T15:24:26.397 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/monmaptool: set fsid to 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:24:26.397 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-15T15:24:26.397 INFO:teuthology.orchestra.run.smithi043.stderr: 2024-05-15T15:24:26.397 INFO:teuthology.orchestra.run.smithi043.stderr:Creating mon... 2024-05-15T15:24:27.302 INFO:teuthology.orchestra.run.smithi043.stderr:create mon.a on 2024-05-15T15:24:27.540 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-05-15T15:24:27.695 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-22d82864-12cf-11ef-bc98-c7b262605968.target → /etc/systemd/system/ceph-22d82864-12cf-11ef-bc98-c7b262605968.target. 2024-05-15T15:24:27.695 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-22d82864-12cf-11ef-bc98-c7b262605968.target → /etc/systemd/system/ceph-22d82864-12cf-11ef-bc98-c7b262605968.target. 2024-05-15T15:24:28.008 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: Failed to reset failed state of unit ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.a.service: Unit ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.a.service not loaded. 2024-05-15T15:24:28.016 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: Created symlink /etc/systemd/system/ceph-22d82864-12cf-11ef-bc98-c7b262605968.target.wants/ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.a.service → /etc/systemd/system/ceph-22d82864-12cf-11ef-bc98-c7b262605968@.service. 2024-05-15T15:24:28.637 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:28 smithi043 podman[24075]: 2024-05-15 15:24:28.596950784 +0000 UTC m=+0.296245922 container start 319b02d29ae26615885f88c836785a84ba4eb9a9a6bec0b6bb02a22076512cb8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, version=8, distribution-scope=public, io.buildah.version=1.19.8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, RELEASE=HEAD, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True) 2024-05-15T15:24:28.637 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:28 smithi043 ceph-mon[24094]: mkfs 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:24:28.674 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: disabled 2024-05-15T15:24:28.681 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: inactive 2024-05-15T15:24:28.682 INFO:teuthology.orchestra.run.smithi043.stderr:firewalld.service is not enabled 2024-05-15T15:24:28.682 INFO:teuthology.orchestra.run.smithi043.stderr:Not possible to enable service . firewalld.service is not available 2024-05-15T15:24:28.683 INFO:teuthology.orchestra.run.smithi043.stderr:Waiting for mon to start... 2024-05-15T15:24:28.683 INFO:teuthology.orchestra.run.smithi043.stderr:Waiting for mon... 2024-05-15T15:24:28.908 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:28 smithi043 ceph-mon[24094]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-15T15:24:28.908 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:28 smithi043 bash[24075]: 319b02d29ae26615885f88c836785a84ba4eb9a9a6bec0b6bb02a22076512cb8 2024-05-15T15:24:28.908 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:28 smithi043 systemd[1]: Started Ceph mon.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:24:29.204 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: cluster: 2024-05-15T15:24:29.204 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: id: 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:24:29.204 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-05-15T15:24:29.204 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:29.204 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: services: 2024-05-15T15:24:29.204 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.567043s) 2024-05-15T15:24:29.205 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mgr: no daemons active 2024-05-15T15:24:29.205 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-05-15T15:24:29.205 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:29.205 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: data: 2024-05-15T15:24:29.205 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-05-15T15:24:29.205 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-05-15T15:24:29.205 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-05-15T15:24:29.205 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: pgs: 2024-05-15T15:24:29.205 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:29.737 INFO:teuthology.orchestra.run.smithi043.stderr:mon is available 2024-05-15T15:24:29.737 INFO:teuthology.orchestra.run.smithi043.stderr:Assimilating anything we can from ceph.conf... 2024-05-15T15:24:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:29 smithi043 ceph-mon[24094]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-15T15:24:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:29 smithi043 ceph-mon[24094]: monmap e1: 1 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0]} 2024-05-15T15:24:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:29 smithi043 ceph-mon[24094]: fsmap 2024-05-15T15:24:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:29 smithi043 ceph-mon[24094]: osdmap e1: 0 total, 0 up, 0 in 2024-05-15T15:24:29.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:29 smithi043 ceph-mon[24094]: mgrmap e1: no daemons active 2024-05-15T15:24:29.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:29 smithi043 ceph-mon[24094]: from='client.? 172.21.15.43:0/1840441932' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:24:30.230 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:30.230 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: [global] 2024-05-15T15:24:30.230 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: fsid = 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:24:30.230 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.43:3300,v1:172.21.15.43:6789] 2024-05-15T15:24:30.230 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-15T15:24:30.231 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-15T15:24:30.231 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-15T15:24:30.231 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-15T15:24:30.231 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:30.231 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: [mgr] 2024-05-15T15:24:30.232 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-05-15T15:24:30.232 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-15T15:24:30.232 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:30.232 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: [osd] 2024-05-15T15:24:30.232 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-15T15:24:30.232 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-15T15:24:30.724 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:30 smithi043 ceph-mon[24094]: from='client.? 172.21.15.43:0/2872587490' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-15T15:24:30.724 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:30 smithi043 ceph-mon[24094]: from='client.? 172.21.15.43:0/2872587490' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-15T15:24:30.726 INFO:teuthology.orchestra.run.smithi043.stderr:Generating new minimal ceph.conf... 2024-05-15T15:24:31.715 INFO:teuthology.orchestra.run.smithi043.stderr:Restarting the monitor... 2024-05-15T15:24:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:31 smithi043 ceph-mon[24094]: from='client.? 172.21.15.43:0/2213766994' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:24:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:31 smithi043 systemd[1]: Stopping Ceph mon.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:24:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:31 smithi043 bash[24311]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon.a" found: no such container 2024-05-15T15:24:32.215 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a[24090]: 2024-05-15T15:24:31.881+0000 7fc017954700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T15:24:32.216 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a[24090]: 2024-05-15T15:24:31.881+0000 7fc017954700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-05-15T15:24:32.216 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:31 smithi043 podman[24322]: 2024-05-15 15:24:31.922806144 +0000 UTC m=+0.149770851 container died 319b02d29ae26615885f88c836785a84ba4eb9a9a6bec0b6bb02a22076512cb8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.display-name=CentOS Stream 8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_CLEAN=True, ceph=True, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-15T15:24:32.216 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:32 smithi043 podman[24322]: 2024-05-15 15:24:32.214362021 +0000 UTC m=+0.441326727 container cleanup 319b02d29ae26615885f88c836785a84ba4eb9a9a6bec0b6bb02a22076512cb8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, distribution-scope=public, version=8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, GIT_BRANCH=HEAD) 2024-05-15T15:24:32.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:32 smithi043 bash[24322]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a 2024-05-15T15:24:32.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:32 smithi043 bash[24349]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon.a" found: no such container 2024-05-15T15:24:32.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:32 smithi043 podman[24335]: 2024-05-15 15:24:32.407023476 +0000 UTC m=+0.481466929 container remove 319b02d29ae26615885f88c836785a84ba4eb9a9a6bec0b6bb02a22076512cb8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, com.redhat.component=centos-stream-container, release=754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, name=centos-stream, vendor=Red Hat, Inc., architecture=x86_64, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, distribution-scope=public, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8) 2024-05-15T15:24:32.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:32 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.a.service: Deactivated successfully. 2024-05-15T15:24:32.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:32 smithi043 systemd[1]: Stopped Ceph mon.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:24:32.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:32 smithi043 systemd[1]: Starting Ceph mon.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:24:33.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:32 smithi043 podman[24411]: 2024-05-15 15:24:32.546694962 +0000 UTC m=+0.018655427 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-15T15:24:33.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:32 smithi043 podman[24411]: 2024-05-15 15:24:32.765066911 +0000 UTC m=+0.237027393 container create 784dab08799bb027c2a62bbc7910f26443488e270e633f6ab6e404402dc29cf1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, release=754, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.buildah.version=1.19.8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , RELEASE=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8) 2024-05-15T15:24:35.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 podman[24411]: 2024-05-15 15:24:34.616122396 +0000 UTC m=+2.088082851 container init 784dab08799bb027c2a62bbc7910f26443488e270e633f6ab6e404402dc29cf1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_CLEAN=True, version=8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, ceph=True, maintainer=Guillaume Abrioux , io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 podman[24411]: 2024-05-15 15:24:34.620750363 +0000 UTC m=+2.092710827 container start 784dab08799bb027c2a62bbc7910f26443488e270e633f6ab6e404402dc29cf1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, ceph=True, vcs-type=git, io.openshift.expose-services=, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0) 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: set uid:gid to 167:167 (ceph:ceph) 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: pidfile_write: ignore empty --pid-file 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: load: jerasure load: lrc 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: RocksDB version: 6.15.5 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Compile date Apr 18 2022 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: DB SUMMARY 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: DB Session ID: SG94MPN9008VVCOH1WMW 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: CURRENT file: CURRENT 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: IDENTITY file: IDENTITY 2024-05-15T15:24:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82635 ; 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.error_if_exists: 0 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.create_if_missing: 0 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.paranoid_checks: 1 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.env: 0x564b708c6860 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.fs: Posix File System 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.info_log: 0x564b725c7dc0 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_file_opening_threads: 16 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.statistics: (nil) 2024-05-15T15:24:35.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.use_fsync: 0 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_log_file_size: 0 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.keep_log_file_num: 1000 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.recycle_log_file_num: 0 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.allow_fallocate: 1 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.allow_mmap_reads: 0 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.allow_mmap_writes: 0 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.use_direct_reads: 0 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.create_missing_column_families: 0 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.db_log_dir: 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-05-15T15:24:35.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.advise_random_on_open: 1 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.db_write_buffer_size: 0 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.write_buffer_manager: 0x564b726b82a0 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-15T15:24:35.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.rate_limiter: (nil) 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.wal_recovery_mode: 2 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.enable_thread_tracking: 0 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.enable_pipelined_write: 0 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.unordered_write: 0 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.row_cache: None 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.wal_filter: None 2024-05-15T15:24:35.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.allow_ingest_behind: 0 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.preserve_deletes: 0 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.two_write_queues: 0 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.manual_wal_flush: 0 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.atomic_flush: 0 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.log_readahead_size: 0 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-15T15:24:35.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.best_efforts_recovery: 0 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.allow_data_in_errors: 0 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.db_host_id: __hostname__ 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_background_jobs: 2 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_background_compactions: -1 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_subcompactions: 1 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-15T15:24:35.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_total_wal_size: 0 2024-05-15T15:24:35.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_open_files: -1 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bytes_per_sync: 0 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_readahead_size: 0 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_background_flushes: -1 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Compression algorithms supported: 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-15T15:24:35.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: kZSTD supported: 0 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: kXpressCompression supported: 0 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: kLZ4HCCompression supported: 1 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: kLZ4Compression supported: 1 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: kBZip2Compression supported: 0 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: kZlibCompression supported: 1 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: kSnappyCompression supported: 1 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.merge_operator: 2024-05-15T15:24:35.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_filter: None 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_filter_factory: None 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.sst_partitioner_factory: None 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564b72593d00) 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: cache_index_and_filter_blocks: 1 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: pin_top_level_index_and_filter: 1 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: index_type: 0 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: data_block_index_type: 0 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: index_shortening: 1 2024-05-15T15:24:35.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: hash_index_allow_collision: 1 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: checksum: 1 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: no_block_cache: 0 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_cache: 0x564b725fe170 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_cache_name: BinnedLRUCache 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_cache_options: 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: capacity : 536870912 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: num_shard_bits : 4 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: strict_capacity_limit : 0 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: high_pri_pool_ratio: 0.000 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_cache_compressed: (nil) 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: persistent_cache: (nil) 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_size: 4096 2024-05-15T15:24:35.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_size_deviation: 10 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_restart_interval: 16 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: index_block_restart_interval: 1 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: metadata_block_size: 4096 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: partition_filters: 0 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_delta_encoding: 1 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: whole_key_filtering: 1 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: verify_compression: 0 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: read_amp_bytes_per_bit: 0 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: format_version: 4 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: enable_index_compression: 1 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_align: 0 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.write_buffer_size: 33554432 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_write_buffer_number: 2 2024-05-15T15:24:35.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compression: NoCompression 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bottommost_compression: Disabled 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.prefix_extractor: nullptr 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.num_levels: 7 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-15T15:24:35.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compression_opts.level: 32767 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compression_opts.strategy: 0 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compression_opts.enabled: false 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.target_file_size_base: 67108864 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-15T15:24:35.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.arena_block_size: 4194304 2024-05-15T15:24:35.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.disable_auto_compactions: 0 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-15T15:24:35.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.table_properties_collectors: 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.inplace_update_support: 0 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.bloom_locality: 0 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.max_successive_merges: 0 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.paranoid_file_checks: 0 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.force_consistency_checks: 1 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.report_bg_io_stats: 0 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.ttl: 2592000 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.enable_blob_files: false 2024-05-15T15:24:35.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.min_blob_size: 0 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.blob_file_size: 268435456 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715786674651934, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715786674652839, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79185, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77448, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71757, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1715786674, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b5f0a75b-4a50-4018-9a0c-fda0a952ab24", "db_session_id": "SG94MPN9008VVCOH1WMW"}} 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-05-15T15:24:35.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715786674653273, "job": 1, "event": "recovery_finished"} 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x564b725e4a80 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: DB pointer 0x564b725f4000 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** DB Stats ** 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:24:35.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 128.2 0.00 0.00 1 0.001 0 0 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 128.2 0.00 0.00 1 0.001 0 0 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 128.2 0.00 0.00 1 0.001 0 0 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 128.2 0.00 0.00 1 0.001 0 0 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-15T15:24:35.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Cumulative compaction: 0.00 GB write, 19.93 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Interval compaction: 0.00 GB write, 19.93 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 128.2 0.00 0.00 1 0.001 0 0 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 128.2 0.00 0.00 1 0.001 0 0 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-15T15:24:35.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 128.2 0.00 0.00 1 0.001 0 0 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: Cumulative compaction: 0.00 GB write, 19.73 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:24:35.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: starting mon.a rank 0 at public addrs [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] at bind addrs [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: mon.a@-1(???) e1 preinit fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: mon.a@-1(???).mds e1 new map 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: mon.a@-1(???).mds e1 print_map 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout: e1 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout: legacy client fscid: -1 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout: No filesystems configured 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-15T15:24:35.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-15T15:24:35.111 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-15T15:24:35.111 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-15T15:24:35.111 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-05-15T15:24:35.111 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta expand map: {default=false} 2024-05-15T15:24:35.111 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta from 'false' to 'false' 2024-05-15T15:24:35.111 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta expanded map: {default=false} 2024-05-15T15:24:35.111 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta expand map: {default=info} 2024-05-15T15:24:35.111 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta from 'info' to 'info' 2024-05-15T15:24:35.111 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta expanded map: {default=info} 2024-05-15T15:24:35.111 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta expand map: {default=daemon} 2024-05-15T15:24:35.112 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta from 'daemon' to 'daemon' 2024-05-15T15:24:35.112 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta expanded map: {default=daemon} 2024-05-15T15:24:35.112 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta expand map: {default=debug} 2024-05-15T15:24:35.112 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta from 'debug' to 'debug' 2024-05-15T15:24:35.112 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: expand_channel_meta expanded map: {default=debug} 2024-05-15T15:24:35.112 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-15T15:24:35.112 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: monmap e1: 1 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0]} 2024-05-15T15:24:35.112 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: fsmap 2024-05-15T15:24:35.112 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: osdmap e1: 0 total, 0 up, 0 in 2024-05-15T15:24:35.112 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:34 smithi043 ceph-mon[24430]: mgrmap e1: no daemons active 2024-05-15T15:24:35.432 INFO:teuthology.orchestra.run.smithi043.stderr:Setting mon public_network to 172.21.0.0/20 2024-05-15T15:24:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:35 smithi043 bash[24411]: 784dab08799bb027c2a62bbc7910f26443488e270e633f6ab6e404402dc29cf1 2024-05-15T15:24:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:35 smithi043 systemd[1]: Started Ceph mon.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:24:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:39 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2447094510' entity='client.admin' 2024-05-15T15:24:42.192 INFO:teuthology.orchestra.run.smithi043.stderr:Wrote config to /etc/ceph/ceph.conf 2024-05-15T15:24:42.192 INFO:teuthology.orchestra.run.smithi043.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-05-15T15:24:42.192 INFO:teuthology.orchestra.run.smithi043.stderr:Creating mgr... 2024-05-15T15:24:42.193 INFO:teuthology.orchestra.run.smithi043.stderr:Verifying port 9283 ... 2024-05-15T15:24:42.386 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: Failed to reset failed state of unit ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.y.service: Unit ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.y.service not loaded. 2024-05-15T15:24:42.393 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: Created symlink /etc/systemd/system/ceph-22d82864-12cf-11ef-bc98-c7b262605968.target.wants/ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-22d82864-12cf-11ef-bc98-c7b262605968@.service. 2024-05-15T15:24:42.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:42 smithi043 systemd[1]: Starting Ceph mgr.y for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:24:43.008 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:42 smithi043 podman[24641]: 2024-05-15 15:24:42.754831288 +0000 UTC m=+0.096738826 container create faaa138d25eefb4a83dee6dfc6e9cc184466848f46aa7c4cf0cac44c269820f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, GIT_BRANCH=HEAD, distribution-scope=public, vcs-type=git, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., release=754, com.redhat.component=centos-stream-container, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870) 2024-05-15T15:24:43.009 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:42 smithi043 podman[24641]: 2024-05-15 15:24:42.676321739 +0000 UTC m=+0.018229287 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-15T15:24:43.009 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:42 smithi043 podman[24641]: 2024-05-15 15:24:42.949472314 +0000 UTC m=+0.291379853 container init faaa138d25eefb4a83dee6dfc6e9cc184466848f46aa7c4cf0cac44c269820f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, release=754, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, name=centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True) 2024-05-15T15:24:43.009 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:42 smithi043 podman[24641]: 2024-05-15 15:24:42.952462633 +0000 UTC m=+0.294370174 container start faaa138d25eefb4a83dee6dfc6e9cc184466848f46aa7c4cf0cac44c269820f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, RELEASE=HEAD, release=754, distribution-scope=public, version=8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, ceph=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git) 2024-05-15T15:24:43.009 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:43 smithi043 bash[24641]: faaa138d25eefb4a83dee6dfc6e9cc184466848f46aa7c4cf0cac44c269820f3 2024-05-15T15:24:43.021 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: disabled 2024-05-15T15:24:43.029 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: inactive 2024-05-15T15:24:43.030 INFO:teuthology.orchestra.run.smithi043.stderr:firewalld.service is not enabled 2024-05-15T15:24:43.030 INFO:teuthology.orchestra.run.smithi043.stderr:Not possible to enable service . firewalld.service is not available 2024-05-15T15:24:43.038 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: disabled 2024-05-15T15:24:43.045 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: inactive 2024-05-15T15:24:43.046 INFO:teuthology.orchestra.run.smithi043.stderr:firewalld.service is not enabled 2024-05-15T15:24:43.047 INFO:teuthology.orchestra.run.smithi043.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-05-15T15:24:43.047 INFO:teuthology.orchestra.run.smithi043.stderr:Waiting for mgr to start... 2024-05-15T15:24:43.047 INFO:teuthology.orchestra.run.smithi043.stderr:Waiting for mgr... 2024-05-15T15:24:43.327 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:43 smithi043 systemd[1]: Started Ceph mgr.y for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:24:43.327 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:43.105+0000 7f8838460000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:24:43.327 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:43.240+0000 7f8838460000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:24:43.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:43.344+0000 7f8838460000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:24:43.641 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:43.641 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: { 2024-05-15T15:24:43.641 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "fsid": "22d82864-12cf-11ef-bc98-c7b262605968", 2024-05-15T15:24:43.641 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "health": { 2024-05-15T15:24:43.641 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "checks": {}, 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "mutes": [] 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "quorum": [ 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 0 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: ], 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "a" 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: ], 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "monmap": { 2024-05-15T15:24:43.642 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "osdmap": { 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "pgmap": { 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-15T15:24:43.643 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "fsmap": { 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:43.644 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "available": false, 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "modules": [ 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "iostat", 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "nfs", 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "restful" 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: ], 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "services": {} 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "servicemap": { 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "modified": "2024-05-15T15:24:28.633720+0000", 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "services": {} 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:43.645 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-15T15:24:43.646 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: } 2024-05-15T15:24:43.947 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:43 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/997605018' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-15T15:24:43.948 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:43.681+0000 7f8838460000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:24:43.948 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:43.767+0000 7f8838460000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:24:44.242 INFO:teuthology.orchestra.run.smithi043.stderr:mgr not available, waiting (1/15)... 2024-05-15T15:24:45.225 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:44.966+0000 7f8838460000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:24:45.226 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:45.137+0000 7f8838460000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:24:45.226 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:45.223+0000 7f8838460000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:24:45.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:45.554+0000 7f8838460000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:24:45.755 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:45.652+0000 7f8838460000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:24:45.755 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:45.752+0000 7f8838460000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:24:46.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:46.008+0000 7f8838460000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:24:46.700 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:46.486+0000 7f8838460000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:24:46.789 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:46.789 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: { 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "fsid": "22d82864-12cf-11ef-bc98-c7b262605968", 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "health": { 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "checks": {}, 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "mutes": [] 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "quorum": [ 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 0 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: ], 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "a" 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: ], 2024-05-15T15:24:46.790 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "monmap": { 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "osdmap": { 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:46.791 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "pgmap": { 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "fsmap": { 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:46.792 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "available": false, 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "modules": [ 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "iostat", 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "nfs", 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "restful" 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: ], 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "services": {} 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "servicemap": { 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "modified": "2024-05-15T15:24:28.633720+0000", 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "services": {} 2024-05-15T15:24:46.793 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:46.794 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-15T15:24:46.794 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: } 2024-05-15T15:24:47.045 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:46 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2354991238' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-15T15:24:47.335 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:46.999+0000 7f8838460000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:24:47.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:47.095+0000 7f8838460000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:24:47.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:47.403+0000 7f8838460000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:24:47.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:47.495+0000 7f8838460000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:24:48.268 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:47.913+0000 7f8838460000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:24:48.268 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:48 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:48.052+0000 7f8838460000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:24:48.268 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:48 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:48.147+0000 7f8838460000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:24:48.268 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:48 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:48.265+0000 7f8838460000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:24:48.585 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:48 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:48.357+0000 7f8838460000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:24:49.270 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:48 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:48.932+0000 7f8838460000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:24:49.273 INFO:teuthology.orchestra.run.smithi043.stderr:mgr not available, waiting (2/15)... 2024-05-15T15:24:49.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:49.311+0000 7f8838460000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:24:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: Activating manager daemon y 2024-05-15T15:24:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: mgrmap e2: y(active, starting, since 0.00284781s) 2024-05-15T15:24:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:24:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:24:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:24:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:24:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:24:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: Manager daemon y is now available 2024-05-15T15:24:49.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:24:49.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:24:49.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' 2024-05-15T15:24:49.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' 2024-05-15T15:24:49.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:49 smithi043 ceph-mon[24430]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' 2024-05-15T15:24:51.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:51 smithi043 ceph-mon[24430]: mgrmap e3: y(active, since 1.00624s) 2024-05-15T15:24:52.015 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:52.015 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: { 2024-05-15T15:24:52.015 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "fsid": "22d82864-12cf-11ef-bc98-c7b262605968", 2024-05-15T15:24:52.015 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "health": { 2024-05-15T15:24:52.015 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-15T15:24:52.015 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "checks": {}, 2024-05-15T15:24:52.015 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "mutes": [] 2024-05-15T15:24:52.015 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:52.015 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "quorum": [ 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 0 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: ], 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "a" 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: ], 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "quorum_age": 17, 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "monmap": { 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "osdmap": { 2024-05-15T15:24:52.016 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "pgmap": { 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-15T15:24:52.017 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "fsmap": { 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "available": true, 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "modules": [ 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "iostat", 2024-05-15T15:24:52.018 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "nfs", 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "restful" 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: ], 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "services": {} 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "servicemap": { 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "modified": "2024-05-15T15:24:28.633720+0000", 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "services": {} 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: }, 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-15T15:24:52.019 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: } 2024-05-15T15:24:52.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:52 smithi043 ceph-mon[24430]: mgrmap e4: y(active, since 2s) 2024-05-15T15:24:52.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:52 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3434218880' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-15T15:24:52.796 INFO:teuthology.orchestra.run.smithi043.stderr:mgr is available 2024-05-15T15:24:53.433 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:53.433 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: [global] 2024-05-15T15:24:53.433 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: fsid = 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:24:53.433 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-15T15:24:53.433 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-15T15:24:53.433 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-15T15:24:53.433 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-15T15:24:53.433 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:53.434 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: [mgr] 2024-05-15T15:24:53.434 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-15T15:24:53.434 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 2024-05-15T15:24:53.434 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: [osd] 2024-05-15T15:24:53.434 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-15T15:24:53.434 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-15T15:24:53.741 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:53 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2281425810' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-15T15:24:53.741 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:53 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2281425810' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-15T15:24:54.003 INFO:teuthology.orchestra.run.smithi043.stderr:Enabling cephadm module... 2024-05-15T15:24:54.996 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:54 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2986798136' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-05-15T15:24:54.997 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ignoring --setuser ceph since I am not root 2024-05-15T15:24:54.997 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ignoring --setgroup ceph since I am not root 2024-05-15T15:24:54.997 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:54.862+0000 7fb663d09000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:24:55.304 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:54.996+0000 7fb663d09000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:24:55.305 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:55.098+0000 7fb663d09000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:24:55.561 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:55.425+0000 7fb663d09000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:24:55.561 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:55.511+0000 7fb663d09000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:24:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:55 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2986798136' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-05-15T15:24:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:55 smithi043 ceph-mon[24430]: mgrmap e5: y(active, since 5s) 2024-05-15T15:24:56.053 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: { 2024-05-15T15:24:56.053 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 5, 2024-05-15T15:24:56.053 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "available": true, 2024-05-15T15:24:56.053 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "active_name": "y", 2024-05-15T15:24:56.053 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-15T15:24:56.053 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: } 2024-05-15T15:24:56.643 INFO:teuthology.orchestra.run.smithi043.stderr:Waiting for the mgr to restart... 2024-05-15T15:24:56.643 INFO:teuthology.orchestra.run.smithi043.stderr:Waiting for mgr epoch 5... 2024-05-15T15:24:56.896 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:24:56 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/4249351224' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-15T15:24:56.896 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:56.719+0000 7fb663d09000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:24:56.896 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:56.894+0000 7fb663d09000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:24:57.310 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:56.981+0000 7fb663d09000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:24:57.310 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:57 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:57.308+0000 7fb663d09000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:24:57.585 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:57 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:57.402+0000 7fb663d09000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:24:57.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:57 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:57.498+0000 7fb663d09000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:24:58.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:57 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:57.748+0000 7fb663d09000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:24:58.585 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:58 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:58.208+0000 7fb663d09000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:24:59.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:58 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:58.711+0000 7fb663d09000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:24:59.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:58 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:58.808+0000 7fb663d09000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:24:59.585 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:59.115+0000 7fb663d09000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:24:59.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:59.207+0000 7fb663d09000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:24:59.979 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:59.624+0000 7fb663d09000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:24:59.979 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:59.763+0000 7fb663d09000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:24:59.979 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:59.859+0000 7fb663d09000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:24:59.979 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:24:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:24:59.977+0000 7fb663d09000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:25:00.335 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:00.068+0000 7fb663d09000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:25:01.017 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:00.639+0000 7fb663d09000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:25:01.335 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:01.014+0000 7fb663d09000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:25:02.079 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: { 2024-05-15T15:25:02.079 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-05-15T15:25:02.080 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "initialized": true 2024-05-15T15:25:02.080 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: } 2024-05-15T15:25:02.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:25:02] ENGINE Bus STARTING 2024-05-15T15:25:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: Active manager daemon y restarted 2024-05-15T15:25:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: Activating manager daemon y 2024-05-15T15:25:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: osdmap e2: 0 total, 0 up, 0 in 2024-05-15T15:25:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: mgrmap e6: y(active, starting, since 0.0524469s) 2024-05-15T15:25:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: Manager daemon y is now available 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:25:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:02 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:25:02.664 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:25:02] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:25:02.664 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:25:02] ENGINE Bus STARTED 2024-05-15T15:25:02.667 INFO:teuthology.orchestra.run.smithi043.stderr:mgr epoch 5 is available 2024-05-15T15:25:02.667 INFO:teuthology.orchestra.run.smithi043.stderr:Setting orchestrator backend to cephadm... 2024-05-15T15:25:03.282 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:03 smithi043 ceph-mon[24430]: mgrmap e7: y(active, since 1.05495s) 2024-05-15T15:25:03.282 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:03 smithi043 ceph-mon[24430]: [15/May/2024:15:25:02] ENGINE Bus STARTING 2024-05-15T15:25:03.282 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:03 smithi043 ceph-mon[24430]: [15/May/2024:15:25:02] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:25:03.282 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:03 smithi043 ceph-mon[24430]: [15/May/2024:15:25:02] ENGINE Bus STARTED 2024-05-15T15:25:03.282 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:03 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:03.282 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:03 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:04.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:04 smithi043 ceph-mon[24430]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:04.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:04 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:04.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:04 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:04.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:04 smithi043 ceph-mon[24430]: mgrmap e8: y(active, since 2s) 2024-05-15T15:25:04.470 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: value unchanged 2024-05-15T15:25:05.073 INFO:teuthology.orchestra.run.smithi043.stderr:Generating ssh key... 2024-05-15T15:25:05.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:05 smithi043 ceph-mon[24430]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:06.111 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: Generating public/private rsa key pair. 2024-05-15T15:25:06.111 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: Your identification has been saved in /tmp/tmpkdohw0fx/key. 2024-05-15T15:25:06.111 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: Your public key has been saved in /tmp/tmpkdohw0fx/key.pub. 2024-05-15T15:25:06.112 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: The key fingerprint is: 2024-05-15T15:25:06.112 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: SHA256:J6bppE52fupRZRZnyk3/3g4lRp7+5jW49LdmqRBjDmI ceph-22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:25:06.112 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: The key's randomart image is: 2024-05-15T15:25:06.112 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: +---[RSA 3072]----+ 2024-05-15T15:25:06.112 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: | . + | 2024-05-15T15:25:06.112 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: | . B . | 2024-05-15T15:25:06.112 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: | * . o | 2024-05-15T15:25:06.112 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: | + o o | 2024-05-15T15:25:06.113 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: | ES..+ = o| 2024-05-15T15:25:06.113 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: | .=.o+ oo.+.| 2024-05-15T15:25:06.113 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: | o * o oo.=| 2024-05-15T15:25:06.113 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: | o * .. o oB=| 2024-05-15T15:25:06.113 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: | .o.=o o+=*| 2024-05-15T15:25:06.113 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: +----[SHA256]-----+ 2024-05-15T15:25:06.989 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:06 smithi043 ceph-mon[24430]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:06.989 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:06 smithi043 ceph-mon[24430]: Generating ssh key... 2024-05-15T15:25:06.989 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:06 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:06.989 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:06 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:06.989 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCoOqZ6XHKL+mRv+iKbHtmxACew0KhmDde3xAu/ExhxSWIIZ7jCwA7SjeP8GloaqXGF9kYWUb47UzTnUNBapdzpS7gvZmVaU6QJdToq9zerRmTAlIci4bV3Qbwtd1w8HCfA858Xr7r2GLon8P7YWjlDvmxML1wYfjo51GHlwRX9x4l1zUOdvYcUjgIPhhccCmKPNXj5OVxstWIs6zCr1sApDmxqyUmtJweg24xsF6Mp/Sa1/8uK6COKLDowgxe0v3Nf5AfqiaCUpH8Guj7Oh96QyT988/q4cI/5r2sP/9NAKMNKQfuK99/jZ/OlYm2rW1B6/faEg7FLHlKbH+CUgmwN5cx8Y2SnBf3tDDBbou3iLsV6cwLAkKFkusweAdCxtBkDhM4YiwOVieGos24JoHi3AjzinfLFO9YktKYwzvVsBUVJl+VCh36jcipiXmddiYnbCMuPMVi0c9nybgKJjbM8N40KPWJMc4CYx4NlIKghH8/YojXC6+APs3FRpouBtyk= ceph-22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:25:07.505 INFO:teuthology.orchestra.run.smithi043.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-05-15T15:25:07.505 INFO:teuthology.orchestra.run.smithi043.stderr:Adding key to root@localhost authorized_keys... 2024-05-15T15:25:07.505 INFO:teuthology.orchestra.run.smithi043.stderr:Adding host smithi043... 2024-05-15T15:25:07.895 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:07 smithi043 ceph-mon[24430]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:09.028 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: Added host 'smithi043' with addr '172.21.15.43' 2024-05-15T15:25:09.039 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:08 smithi043 ceph-mon[24430]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi043", "addr": "172.21.15.43", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:09.039 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:08 smithi043 ceph-mon[24430]: Deploying cephadm binary to smithi043 2024-05-15T15:25:09.648 INFO:teuthology.orchestra.run.smithi043.stderr:Deploying unmanaged mon service... 2024-05-15T15:25:10.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:10 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:10.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:10 smithi043 ceph-mon[24430]: Added host smithi043 2024-05-15T15:25:10.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:10 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:10.310 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: Scheduled mon update... 2024-05-15T15:25:10.868 INFO:teuthology.orchestra.run.smithi043.stderr:Deploying unmanaged mgr service... 2024-05-15T15:25:11.498 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:11 smithi043 ceph-mon[24430]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:11.498 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:11 smithi043 ceph-mon[24430]: Saving service mon spec with placement count:5 2024-05-15T15:25:11.499 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:11 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:11.501 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-05-15T15:25:12.752 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:12 smithi043 ceph-mon[24430]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:12.752 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:12 smithi043 ceph-mon[24430]: Saving service mgr spec with placement count:2 2024-05-15T15:25:12.753 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:12 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:14.075 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:13 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1717589581' entity='client.admin' 2024-05-15T15:25:14.076 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:13 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:14.076 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:13 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:15 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2839072776' entity='client.admin' 2024-05-15T15:25:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:15 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:15.941 INFO:teuthology.orchestra.run.smithi043.stderr:Enabling the dashboard module... 2024-05-15T15:25:17.762 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:17 smithi043 ceph-mon[24430]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:17.762 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:17 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1077728285' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-05-15T15:25:17.762 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ignoring --setuser ceph since I am not root 2024-05-15T15:25:17.762 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ignoring --setgroup ceph since I am not root 2024-05-15T15:25:17.762 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:17.625+0000 7fd498c45000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:25:17.762 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:17.759+0000 7fd498c45000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:25:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:17.863+0000 7fd498c45000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:25:18.462 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:18.183+0000 7fd498c45000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:25:18.462 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:18.270+0000 7fd498c45000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:25:18.781 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:18 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1077728285' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-05-15T15:25:18.781 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:18 smithi043 ceph-mon[24430]: mgrmap e9: y(active, since 16s) 2024-05-15T15:25:19.040 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: { 2024-05-15T15:25:19.040 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "epoch": 9, 2024-05-15T15:25:19.040 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "available": true, 2024-05-15T15:25:19.040 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "active_name": "y", 2024-05-15T15:25:19.040 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-15T15:25:19.040 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: } 2024-05-15T15:25:19.729 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:19 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3594983288' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-15T15:25:19.730 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:19.469+0000 7fd498c45000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:25:19.730 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:19.640+0000 7fd498c45000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:25:19.730 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:19.726+0000 7fd498c45000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:25:19.946 INFO:teuthology.orchestra.run.smithi043.stderr:Waiting for the mgr to restart... 2024-05-15T15:25:19.946 INFO:teuthology.orchestra.run.smithi043.stderr:Waiting for mgr epoch 9... 2024-05-15T15:25:20.335 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:20 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:20.057+0000 7fd498c45000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:25:20.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:20 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:20.153+0000 7fd498c45000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:25:20.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:20 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:20.252+0000 7fd498c45000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:25:20.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:20 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:20.506+0000 7fd498c45000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:25:21.335 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:20 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:20.967+0000 7fd498c45000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:25:21.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:21.471+0000 7fd498c45000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:25:21.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:21.568+0000 7fd498c45000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:25:22.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:21.874+0000 7fd498c45000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:25:22.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:21.967+0000 7fd498c45000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:25:22.738 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:22.384+0000 7fd498c45000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:25:22.738 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:22.522+0000 7fd498c45000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:25:22.738 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:22.617+0000 7fd498c45000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:25:22.738 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:22.736+0000 7fd498c45000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:25:23.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:22.829+0000 7fd498c45000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:25:23.773 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:23.398+0000 7fd498c45000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:25:24.025 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:25:23.771+0000 7fd498c45000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:25:24.712 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:25:24] ENGINE Bus STARTING 2024-05-15T15:25:24.713 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:25:24] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:25:24.713 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:25:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:25:24] ENGINE Bus STARTED 2024-05-15T15:25:24.837 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: { 2024-05-15T15:25:24.837 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-05-15T15:25:24.837 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: "initialized": true 2024-05-15T15:25:24.837 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: } 2024-05-15T15:25:25.027 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: Active manager daemon y restarted 2024-05-15T15:25:25.027 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: Activating manager daemon y 2024-05-15T15:25:25.027 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: osdmap e3: 0 total, 0 up, 0 in 2024-05-15T15:25:25.027 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: mgrmap e10: y(active, starting, since 0.052779s) 2024-05-15T15:25:25.027 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:25:25.027 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:25:25.027 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:25:25.027 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:25:25.028 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:25:25.028 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: Manager daemon y is now available 2024-05-15T15:25:25.028 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:25.028 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:25.028 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:25:25.028 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:25:25.028 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: [15/May/2024:15:25:24] ENGINE Bus STARTING 2024-05-15T15:25:25.028 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:25.831 INFO:teuthology.orchestra.run.smithi043.stderr:mgr epoch 9 is available 2024-05-15T15:25:25.831 INFO:teuthology.orchestra.run.smithi043.stderr:Generating a dashboard self-signed certificate... 2024-05-15T15:25:26.080 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:25 smithi043 ceph-mon[24430]: [15/May/2024:15:25:24] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:25:26.080 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:25 smithi043 ceph-mon[24430]: [15/May/2024:15:25:24] ENGINE Bus STARTED 2024-05-15T15:25:26.080 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:25 smithi043 ceph-mon[24430]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-05-15T15:25:26.080 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:25 smithi043 ceph-mon[24430]: mgrmap e11: y(active, since 1.05509s) 2024-05-15T15:25:26.080 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:25 smithi043 ceph-mon[24430]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-05-15T15:25:26.080 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:26.602 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: Self-signed certificate created 2024-05-15T15:25:27.279 INFO:teuthology.orchestra.run.smithi043.stderr:Creating initial admin user... 2024-05-15T15:25:27.511 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:27 smithi043 ceph-mon[24430]: mgrmap e12: y(active, since 2s) 2024-05-15T15:25:27.511 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:27 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:27.511 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:27 smithi043 ceph-mon[24430]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:27.511 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:27 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:27.511 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:27 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:28.153 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$H6cs9Mvdzz4b9tE/9/NM8.kGRAFYh.dauUksP.rf2qjr63TyEpp1S", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1715786728, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-05-15T15:25:28.897 INFO:teuthology.orchestra.run.smithi043.stderr:Fetching dashboard port number... 2024-05-15T15:25:29.449 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:29 smithi043 ceph-mon[24430]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:29.449 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:29 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:30.031 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: 8443 2024-05-15T15:25:30.761 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:30 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2422268425' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-05-15T15:25:30.775 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: disabled 2024-05-15T15:25:30.783 INFO:teuthology.orchestra.run.smithi043.stderr:systemctl: inactive 2024-05-15T15:25:30.784 INFO:teuthology.orchestra.run.smithi043.stderr:firewalld.service is not enabled 2024-05-15T15:25:30.784 INFO:teuthology.orchestra.run.smithi043.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-05-15T15:25:30.786 INFO:teuthology.orchestra.run.smithi043.stderr:Ceph Dashboard is now available at: 2024-05-15T15:25:30.787 INFO:teuthology.orchestra.run.smithi043.stderr: 2024-05-15T15:25:30.787 INFO:teuthology.orchestra.run.smithi043.stderr: URL: https://smithi043.front.sepia.ceph.com:8443/ 2024-05-15T15:25:30.787 INFO:teuthology.orchestra.run.smithi043.stderr: User: admin 2024-05-15T15:25:30.787 INFO:teuthology.orchestra.run.smithi043.stderr: Password: ni0b4e39et 2024-05-15T15:25:30.787 INFO:teuthology.orchestra.run.smithi043.stderr: 2024-05-15T15:25:30.787 INFO:teuthology.orchestra.run.smithi043.stderr:Enabling autotune for osd_memory_target 2024-05-15T15:25:31.708 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:31 smithi043 ceph-mon[24430]: mgrmap e13: y(active, since 6s) 2024-05-15T15:25:32.773 INFO:teuthology.orchestra.run.smithi043.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-05-15T15:25:33.360 INFO:teuthology.orchestra.run.smithi043.stderr:You can access the Ceph CLI with: 2024-05-15T15:25:33.360 INFO:teuthology.orchestra.run.smithi043.stderr: 2024-05-15T15:25:33.360 INFO:teuthology.orchestra.run.smithi043.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-05-15T15:25:33.360 INFO:teuthology.orchestra.run.smithi043.stderr: 2024-05-15T15:25:33.360 INFO:teuthology.orchestra.run.smithi043.stderr:Please consider enabling telemetry to help improve Ceph: 2024-05-15T15:25:33.360 INFO:teuthology.orchestra.run.smithi043.stderr: 2024-05-15T15:25:33.360 INFO:teuthology.orchestra.run.smithi043.stderr: ceph telemetry on 2024-05-15T15:25:33.360 INFO:teuthology.orchestra.run.smithi043.stderr: 2024-05-15T15:25:33.360 INFO:teuthology.orchestra.run.smithi043.stderr:For more information see: 2024-05-15T15:25:33.360 INFO:teuthology.orchestra.run.smithi043.stderr: 2024-05-15T15:25:33.361 INFO:teuthology.orchestra.run.smithi043.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-05-15T15:25:33.361 INFO:teuthology.orchestra.run.smithi043.stderr: 2024-05-15T15:25:33.361 INFO:teuthology.orchestra.run.smithi043.stderr:Bootstrap complete. 2024-05-15T15:25:33.398 INFO:tasks.cephadm:Fetching config... 2024-05-15T15:25:33.398 DEBUG:teuthology.orchestra.run.smithi043:> set -ex 2024-05-15T15:25:33.398 DEBUG:teuthology.orchestra.run.smithi043:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-05-15T15:25:33.417 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-05-15T15:25:33.417 DEBUG:teuthology.orchestra.run.smithi043:> set -ex 2024-05-15T15:25:33.417 DEBUG:teuthology.orchestra.run.smithi043:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-05-15T15:25:33.473 INFO:tasks.cephadm:Fetching mon keyring... 2024-05-15T15:25:33.474 DEBUG:teuthology.orchestra.run.smithi043:> set -ex 2024-05-15T15:25:33.474 DEBUG:teuthology.orchestra.run.smithi043:> sudo dd if=/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/keyring of=/dev/stdout 2024-05-15T15:25:33.540 INFO:tasks.cephadm:Fetching pub ssh key... 2024-05-15T15:25:33.540 DEBUG:teuthology.orchestra.run.smithi043:> set -ex 2024-05-15T15:25:33.541 DEBUG:teuthology.orchestra.run.smithi043:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-05-15T15:25:33.597 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-05-15T15:25:33.598 DEBUG:teuthology.orchestra.run.smithi043:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCoOqZ6XHKL+mRv+iKbHtmxACew0KhmDde3xAu/ExhxSWIIZ7jCwA7SjeP8GloaqXGF9kYWUb47UzTnUNBapdzpS7gvZmVaU6QJdToq9zerRmTAlIci4bV3Qbwtd1w8HCfA858Xr7r2GLon8P7YWjlDvmxML1wYfjo51GHlwRX9x4l1zUOdvYcUjgIPhhccCmKPNXj5OVxstWIs6zCr1sApDmxqyUmtJweg24xsF6Mp/Sa1/8uK6COKLDowgxe0v3Nf5AfqiaCUpH8Guj7Oh96QyT988/q4cI/5r2sP/9NAKMNKQfuK99/jZ/OlYm2rW1B6/faEg7FLHlKbH+CUgmwN5cx8Y2SnBf3tDDBbou3iLsV6cwLAkKFkusweAdCxtBkDhM4YiwOVieGos24JoHi3AjzinfLFO9YktKYwzvVsBUVJl+VCh36jcipiXmddiYnbCMuPMVi0c9nybgKJjbM8N40KPWJMc4CYx4NlIKghH8/YojXC6+APs3FRpouBtyk= ceph-22d82864-12cf-11ef-bc98-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-15T15:25:33.676 INFO:teuthology.orchestra.run.smithi043.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCoOqZ6XHKL+mRv+iKbHtmxACew0KhmDde3xAu/ExhxSWIIZ7jCwA7SjeP8GloaqXGF9kYWUb47UzTnUNBapdzpS7gvZmVaU6QJdToq9zerRmTAlIci4bV3Qbwtd1w8HCfA858Xr7r2GLon8P7YWjlDvmxML1wYfjo51GHlwRX9x4l1zUOdvYcUjgIPhhccCmKPNXj5OVxstWIs6zCr1sApDmxqyUmtJweg24xsF6Mp/Sa1/8uK6COKLDowgxe0v3Nf5AfqiaCUpH8Guj7Oh96QyT988/q4cI/5r2sP/9NAKMNKQfuK99/jZ/OlYm2rW1B6/faEg7FLHlKbH+CUgmwN5cx8Y2SnBf3tDDBbou3iLsV6cwLAkKFkusweAdCxtBkDhM4YiwOVieGos24JoHi3AjzinfLFO9YktKYwzvVsBUVJl+VCh36jcipiXmddiYnbCMuPMVi0c9nybgKJjbM8N40KPWJMc4CYx4NlIKghH8/YojXC6+APs3FRpouBtyk= ceph-22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:25:33.688 DEBUG:teuthology.orchestra.run.smithi053:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCoOqZ6XHKL+mRv+iKbHtmxACew0KhmDde3xAu/ExhxSWIIZ7jCwA7SjeP8GloaqXGF9kYWUb47UzTnUNBapdzpS7gvZmVaU6QJdToq9zerRmTAlIci4bV3Qbwtd1w8HCfA858Xr7r2GLon8P7YWjlDvmxML1wYfjo51GHlwRX9x4l1zUOdvYcUjgIPhhccCmKPNXj5OVxstWIs6zCr1sApDmxqyUmtJweg24xsF6Mp/Sa1/8uK6COKLDowgxe0v3Nf5AfqiaCUpH8Guj7Oh96QyT988/q4cI/5r2sP/9NAKMNKQfuK99/jZ/OlYm2rW1B6/faEg7FLHlKbH+CUgmwN5cx8Y2SnBf3tDDBbou3iLsV6cwLAkKFkusweAdCxtBkDhM4YiwOVieGos24JoHi3AjzinfLFO9YktKYwzvVsBUVJl+VCh36jcipiXmddiYnbCMuPMVi0c9nybgKJjbM8N40KPWJMc4CYx4NlIKghH8/YojXC6+APs3FRpouBtyk= ceph-22d82864-12cf-11ef-bc98-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-15T15:25:33.730 INFO:teuthology.orchestra.run.smithi053.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCoOqZ6XHKL+mRv+iKbHtmxACew0KhmDde3xAu/ExhxSWIIZ7jCwA7SjeP8GloaqXGF9kYWUb47UzTnUNBapdzpS7gvZmVaU6QJdToq9zerRmTAlIci4bV3Qbwtd1w8HCfA858Xr7r2GLon8P7YWjlDvmxML1wYfjo51GHlwRX9x4l1zUOdvYcUjgIPhhccCmKPNXj5OVxstWIs6zCr1sApDmxqyUmtJweg24xsF6Mp/Sa1/8uK6COKLDowgxe0v3Nf5AfqiaCUpH8Guj7Oh96QyT988/q4cI/5r2sP/9NAKMNKQfuK99/jZ/OlYm2rW1B6/faEg7FLHlKbH+CUgmwN5cx8Y2SnBf3tDDBbou3iLsV6cwLAkKFkusweAdCxtBkDhM4YiwOVieGos24JoHi3AjzinfLFO9YktKYwzvVsBUVJl+VCh36jcipiXmddiYnbCMuPMVi0c9nybgKJjbM8N40KPWJMc4CYx4NlIKghH8/YojXC6+APs3FRpouBtyk= ceph-22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:25:33.744 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-05-15T15:25:33.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:33 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1647747360' entity='client.admin' 2024-05-15T15:25:36.440 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:36.440 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi043", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:25:36.441 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:36.441 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:36.441 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:37.465 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-05-15T15:25:37.465 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-05-15T15:25:38.117 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:37 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3103695084' entity='client.admin' 2024-05-15T15:25:38.117 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:37 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:38.117 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:37 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:40.179 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi053 2024-05-15T15:25:40.179 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-05-15T15:25:40.179 DEBUG:teuthology.orchestra.run.smithi053:> dd of=/etc/ceph/ceph.conf 2024-05-15T15:25:40.199 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-05-15T15:25:40.199 DEBUG:teuthology.orchestra.run.smithi053:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:25:40.258 INFO:tasks.cephadm:Adding host smithi053 to orchestrator... 2024-05-15T15:25:40.258 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch host add smithi053 2024-05-15T15:25:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:40 smithi043 ceph-mon[24430]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:40 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:40 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:40 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:40 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:25:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:40 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:25:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:40 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:25:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:40 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:40 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:42 smithi043 ceph-mon[24430]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi053", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:42 smithi043 ceph-mon[24430]: Deploying cephadm binary to smithi053 2024-05-15T15:25:43.023 INFO:teuthology.orchestra.run.smithi043.stdout:Added host 'smithi053' with addr '172.21.15.53' 2024-05-15T15:25:43.698 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch host ls --format=json 2024-05-15T15:25:44.295 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:44 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:44.296 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:44 smithi043 ceph-mon[24430]: Added host smithi053 2024-05-15T15:25:44.296 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:44 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:44.296 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:44 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:44.296 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:44 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:25:44.296 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:44 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:44.296 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:44 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:45.184 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:45 smithi043 ceph-mon[24430]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:45.184 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:45 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:45.592 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:25:45.592 INFO:teuthology.orchestra.run.smithi043.stdout:[{"addr": "172.21.15.43", "hostname": "smithi043", "labels": [], "status": ""}, {"addr": "172.21.15.53", "hostname": "smithi053", "labels": [], "status": ""}] 2024-05-15T15:25:46.141 INFO:tasks.cephadm:Setting crush tunables to default 2024-05-15T15:25:46.141 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd crush tunables default 2024-05-15T15:25:47.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:47 smithi043 ceph-mon[24430]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-15T15:25:47.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:47 smithi043 ceph-mon[24430]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:48.372 INFO:teuthology.orchestra.run.smithi043.stderr:adjusted tunables profile to default 2024-05-15T15:25:48.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:48 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:48.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:48 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:48.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:48 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi053", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:25:48.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:48 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:48.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:48 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/91864548' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-15T15:25:49.014 INFO:tasks.cephadm:Adding mon.a on smithi043 2024-05-15T15:25:49.014 INFO:tasks.cephadm:Adding mon.c on smithi043 2024-05-15T15:25:49.014 INFO:tasks.cephadm:Adding mon.b on smithi053 2024-05-15T15:25:49.017 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch apply mon '3;smithi043:172.21.15.43=a;smithi043:[v2:172.21.15.43:3301,v1:172.21.15.43:6790]=c;smithi053:172.21.15.53=b' 2024-05-15T15:25:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:49 smithi043 ceph-mon[24430]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:49 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/91864548' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-15T15:25:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:49 smithi043 ceph-mon[24430]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:25:50.729 INFO:teuthology.orchestra.run.smithi053.stdout:Scheduled mon update... 2024-05-15T15:25:51.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:50 smithi043 ceph-mon[24430]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:51.403 DEBUG:teuthology.orchestra.run.smithi043:mon.c> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.c.service 2024-05-15T15:25:51.406 DEBUG:teuthology.orchestra.run.smithi053:mon.b> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.b.service 2024-05-15T15:25:51.409 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-15T15:25:51.409 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph mon dump -f json 2024-05-15T15:25:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi043:172.21.15.43=a;smithi043:[v2:172.21.15.43:3301,v1:172.21.15.43:6790]=c;smithi053:172.21.15.53=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: Saving service mon spec with placement smithi043:172.21.15.43=a;smithi043:[v2:172.21.15.43:3301,v1:172.21.15.43:6790]=c;smithi053:172.21.15.53=b;count:3 2024-05-15T15:25:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:25:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:25:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:25:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:25:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:51 smithi043 ceph-mon[24430]: Deploying daemon mon.c on smithi043 2024-05-15T15:25:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:52 smithi043 ceph-mon[24430]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:53.219 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-05-15T15:25:53.219 INFO:teuthology.orchestra.run.smithi053.stdout:{"epoch":1,"fsid":"22d82864-12cf-11ef-bc98-c7b262605968","modified":"2024-05-15T15:24:25.819270Z","created":"2024-05-15T15:24:25.819270Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:3300","nonce":0},{"type":"v1","addr":"172.21.15.43:6789","nonce":0}]},"addr":"172.21.15.43:6789/0","public_addr":"172.21.15.43:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-15T15:25:53.222 INFO:teuthology.orchestra.run.smithi053.stderr:dumped monmap epoch 1 2024-05-15T15:25:53.996 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:53 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/50681785' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-15T15:25:53.996 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:53 smithi043 systemd[1]: Starting Ceph mon.c for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:25:54.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:53 smithi043 podman[29108]: 2024-05-15 15:25:53.898047442 +0000 UTC m=+0.018199207 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-15T15:25:54.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:53 smithi043 podman[29108]: 2024-05-15 15:25:53.995233862 +0000 UTC m=+0.115385629 container create 44c487fc3e4119314217a291c2701b77ad9418806db84eec9194faf234830d20 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.display-name=CentOS Stream 8, release=754, RELEASE=HEAD, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0) 2024-05-15T15:25:54.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 podman[29108]: 2024-05-15 15:25:54.197607413 +0000 UTC m=+0.317759178 container init 44c487fc3e4119314217a291c2701b77ad9418806db84eec9194faf234830d20 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, vendor=Red Hat, Inc., version=8, vcs-type=git, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, distribution-scope=public) 2024-05-15T15:25:54.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 podman[29108]: 2024-05-15 15:25:54.201231682 +0000 UTC m=+0.321383447 container start 44c487fc3e4119314217a291c2701b77ad9418806db84eec9194faf234830d20 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, io.openshift.expose-services=, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vendor=Red Hat, Inc., RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, name=centos-stream, io.openshift.tags=base centos centos-stream, ceph=True, release=754, version=8, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-15T15:25:54.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: set uid:gid to 167:167 (ceph:ceph) 2024-05-15T15:25:54.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-15T15:25:54.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: pidfile_write: ignore empty --pid-file 2024-05-15T15:25:54.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: load: jerasure load: lrc 2024-05-15T15:25:54.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: RocksDB version: 6.15.5 2024-05-15T15:25:54.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-15T15:25:54.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Compile date Apr 18 2022 2024-05-15T15:25:54.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: DB SUMMARY 2024-05-15T15:25:54.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: DB Session ID: KF1U0DWZUB0Y8NBAI916 2024-05-15T15:25:54.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: CURRENT file: CURRENT 2024-05-15T15:25:54.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: IDENTITY file: IDENTITY 2024-05-15T15:25:54.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-05-15T15:25:54.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-05-15T15:25:54.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-05-15T15:25:54.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.error_if_exists: 0 2024-05-15T15:25:54.249 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.create_if_missing: 0 2024-05-15T15:25:54.249 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.paranoid_checks: 1 2024-05-15T15:25:54.249 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-15T15:25:54.249 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.env: 0x557de28c6860 2024-05-15T15:25:54.249 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.fs: Posix File System 2024-05-15T15:25:54.249 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.info_log: 0x557de427fdc0 2024-05-15T15:25:54.249 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_file_opening_threads: 16 2024-05-15T15:25:54.249 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.statistics: (nil) 2024-05-15T15:25:54.249 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.use_fsync: 0 2024-05-15T15:25:54.251 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_log_file_size: 0 2024-05-15T15:25:54.251 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-15T15:25:54.251 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-15T15:25:54.252 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.keep_log_file_num: 1000 2024-05-15T15:25:54.252 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.recycle_log_file_num: 0 2024-05-15T15:25:54.252 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.allow_fallocate: 1 2024-05-15T15:25:54.252 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.allow_mmap_reads: 0 2024-05-15T15:25:54.252 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.allow_mmap_writes: 0 2024-05-15T15:25:54.252 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.use_direct_reads: 0 2024-05-15T15:25:54.252 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-15T15:25:54.252 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.create_missing_column_families: 0 2024-05-15T15:25:54.253 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.db_log_dir: 2024-05-15T15:25:54.253 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-05-15T15:25:54.253 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-15T15:25:54.253 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-15T15:25:54.253 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-15T15:25:54.253 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-15T15:25:54.253 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-15T15:25:54.253 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-15T15:25:54.253 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.advise_random_on_open: 1 2024-05-15T15:25:54.253 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.db_write_buffer_size: 0 2024-05-15T15:25:54.254 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.write_buffer_manager: 0x557de43702d0 2024-05-15T15:25:54.254 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-15T15:25:54.254 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-15T15:25:54.254 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-15T15:25:54.254 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-15T15:25:54.254 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.rate_limiter: (nil) 2024-05-15T15:25:54.254 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.wal_recovery_mode: 2 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.enable_thread_tracking: 0 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.enable_pipelined_write: 0 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.unordered_write: 0 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.row_cache: None 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.wal_filter: None 2024-05-15T15:25:54.255 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.allow_ingest_behind: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.preserve_deletes: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.two_write_queues: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.manual_wal_flush: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.atomic_flush: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.log_readahead_size: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.best_efforts_recovery: 0 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-15T15:25:54.256 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.allow_data_in_errors: 0 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.db_host_id: __hostname__ 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_background_jobs: 2 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_background_compactions: -1 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_subcompactions: 1 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_total_wal_size: 0 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-15T15:25:54.257 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-15T15:25:54.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-15T15:25:54.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_open_files: -1 2024-05-15T15:25:54.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bytes_per_sync: 0 2024-05-15T15:25:54.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-15T15:25:54.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-15T15:25:54.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_readahead_size: 0 2024-05-15T15:25:54.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_background_flushes: -1 2024-05-15T15:25:54.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Compression algorithms supported: 2024-05-15T15:25:54.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: kZSTD supported: 0 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: kXpressCompression supported: 0 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: kLZ4HCCompression supported: 1 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: kLZ4Compression supported: 1 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: kBZip2Compression supported: 0 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: kZlibCompression supported: 1 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: kSnappyCompression supported: 1 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-15T15:25:54.259 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.merge_operator: 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_filter: None 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_filter_factory: None 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.sst_partitioner_factory: None 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557de424bd00) 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: cache_index_and_filter_blocks: 1 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: pin_top_level_index_and_filter: 1 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: index_type: 0 2024-05-15T15:25:54.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: data_block_index_type: 0 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: index_shortening: 1 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: hash_index_allow_collision: 1 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: checksum: 1 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: no_block_cache: 0 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_cache: 0x557de42b6170 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_cache_name: BinnedLRUCache 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_cache_options: 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: capacity : 536870912 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: num_shard_bits : 4 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: strict_capacity_limit : 0 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: high_pri_pool_ratio: 0.000 2024-05-15T15:25:54.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_cache_compressed: (nil) 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: persistent_cache: (nil) 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_size: 4096 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_size_deviation: 10 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_restart_interval: 16 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: index_block_restart_interval: 1 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: metadata_block_size: 4096 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: partition_filters: 0 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_delta_encoding: 1 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: whole_key_filtering: 1 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: verify_compression: 0 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: read_amp_bytes_per_bit: 0 2024-05-15T15:25:54.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: format_version: 4 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout: enable_index_compression: 1 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_align: 0 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.write_buffer_size: 33554432 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_write_buffer_number: 2 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compression: NoCompression 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bottommost_compression: Disabled 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.prefix_extractor: nullptr 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.num_levels: 7 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-15T15:25:54.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compression_opts.level: 32767 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compression_opts.strategy: 0 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-15T15:25:54.264 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compression_opts.enabled: false 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.target_file_size_base: 67108864 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-15T15:25:54.265 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.arena_block_size: 4194304 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.disable_auto_compactions: 0 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-15T15:25:54.266 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.table_properties_collectors: 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.inplace_update_support: 0 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-15T15:25:54.267 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.bloom_locality: 0 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.max_successive_merges: 0 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.paranoid_file_checks: 0 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.force_consistency_checks: 1 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.report_bg_io_stats: 0 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.ttl: 2592000 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.enable_blob_files: false 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.min_blob_size: 0 2024-05-15T15:25:54.268 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.blob_file_size: 268435456 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715786754228330, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715786754228639, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1715786754, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "f76b58c3-f112-41c9-bd02-b9dd94378f6c", "db_session_id": "KF1U0DWZUB0Y8NBAI916"}} 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-05-15T15:25:54.269 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715786754228961, "job": 1, "event": "recovery_finished"} 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x557de429ca80 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: DB pointer 0x557de42ac000 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** DB Stats ** 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-15T15:25:54.270 INFO:journalctl@ceph.mon.c.smithi043.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:25:54.271 INFO:journalctl@ceph.mon.c.smithi043.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: Interval compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-15T15:25:54.272 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-05-15T15:25:54.273 INFO:journalctl@ceph.mon.c.smithi043.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: using public_addrv [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] 2024-05-15T15:25:54.274 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: starting mon.c rank -1 at public addrs [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] at bind addrs [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(???) e0 preinit fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).mds e1 new map 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).mds e1 print_map 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout: e1 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout: legacy client fscid: -1 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout: No filesystems configured 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-05-15T15:25:54.275 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-05-15T15:25:54.276 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-05-15T15:25:54.276 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-05-15T15:25:54.276 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-05-15T15:25:54.276 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-05-15T15:25:54.276 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-15T15:25:54.276 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-15T15:25:54.276 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-15T15:25:54.276 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-15T15:25:54.276 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mkfs 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:25:54.276 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: monmap e1: 1 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0]} 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: fsmap 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: osdmap e1: 0 total, 0 up, 0 in 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e1: no daemons active 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1840441932' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2872587490' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2872587490' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2213766994' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: monmap e1: 1 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0]} 2024-05-15T15:25:54.277 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: fsmap 2024-05-15T15:25:54.278 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: osdmap e1: 0 total, 0 up, 0 in 2024-05-15T15:25:54.278 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e1: no daemons active 2024-05-15T15:25:54.278 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2447094510' entity='client.admin' 2024-05-15T15:25:54.278 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/997605018' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-15T15:25:54.278 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2354991238' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-15T15:25:54.278 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Activating manager daemon y 2024-05-15T15:25:54.278 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e2: y(active, starting, since 0.00284781s) 2024-05-15T15:25:54.278 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:25:54.278 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:25:54.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:25:54.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:25:54.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:25:54.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Manager daemon y is now available 2024-05-15T15:25:54.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:25:54.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:25:54.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' 2024-05-15T15:25:54.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' 2024-05-15T15:25:54.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14100 172.21.15.43:0/3119026702' entity='mgr.y' 2024-05-15T15:25:54.280 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e3: y(active, since 1.00624s) 2024-05-15T15:25:54.280 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e4: y(active, since 2s) 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3434218880' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2281425810' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2281425810' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2986798136' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2986798136' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e5: y(active, since 5s) 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/4249351224' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Active manager daemon y restarted 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Activating manager daemon y 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: osdmap e2: 0 total, 0 up, 0 in 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e6: y(active, starting, since 0.0524469s) 2024-05-15T15:25:54.281 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Manager daemon y is now available 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:25:54.282 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:25:54.283 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e7: y(active, since 1.05495s) 2024-05-15T15:25:54.283 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: [15/May/2024:15:25:02] ENGINE Bus STARTING 2024-05-15T15:25:54.283 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: [15/May/2024:15:25:02] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:25:54.283 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: [15/May/2024:15:25:02] ENGINE Bus STARTED 2024-05-15T15:25:54.283 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e8: y(active, since 2s) 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Generating ssh key... 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.284 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi043", "addr": "172.21.15.43", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.285 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Deploying cephadm binary to smithi043 2024-05-15T15:25:54.285 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.285 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Added host smithi043 2024-05-15T15:25:54.285 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.285 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.285 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Saving service mon spec with placement count:5 2024-05-15T15:25:54.285 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.285 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Saving service mgr spec with placement count:2 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1717589581' entity='client.admin' 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2839072776' entity='client.admin' 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14118 172.21.15.43:0/1303660855' entity='mgr.y' 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1077728285' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1077728285' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e9: y(active, since 16s) 2024-05-15T15:25:54.286 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3594983288' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-15T15:25:54.287 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Active manager daemon y restarted 2024-05-15T15:25:54.287 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Activating manager daemon y 2024-05-15T15:25:54.287 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: osdmap e3: 0 total, 0 up, 0 in 2024-05-15T15:25:54.287 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e10: y(active, starting, since 0.052779s) 2024-05-15T15:25:54.287 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:25:54.287 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:25:54.287 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:25:54.287 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:25:54.288 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:25:54.288 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Manager daemon y is now available 2024-05-15T15:25:54.288 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.288 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.288 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:25:54.288 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:25:54.288 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: [15/May/2024:15:25:24] ENGINE Bus STARTING 2024-05-15T15:25:54.288 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.288 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: [15/May/2024:15:25:24] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:25:54.288 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: [15/May/2024:15:25:24] ENGINE Bus STARTED 2024-05-15T15:25:54.289 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-05-15T15:25:54.289 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e11: y(active, since 1.05509s) 2024-05-15T15:25:54.289 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-05-15T15:25:54.289 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.289 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e12: y(active, since 2s) 2024-05-15T15:25:54.289 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.289 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.290 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.290 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.290 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.290 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.290 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2422268425' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-05-15T15:25:54.290 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mgrmap e13: y(active, since 6s) 2024-05-15T15:25:54.290 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1647747360' entity='client.admin' 2024-05-15T15:25:54.290 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.291 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi043", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:25:54.291 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.291 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.291 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.291 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3103695084' entity='client.admin' 2024-05-15T15:25:54.291 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.291 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.291 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.291 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.292 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.292 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:54.292 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:25:54.292 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:25:54.292 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:25:54.292 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.292 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.292 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi053", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.293 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Deploying cephadm binary to smithi053 2024-05-15T15:25:54.293 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.293 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Added host smithi053 2024-05-15T15:25:54.293 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.293 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:54.293 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:25:54.293 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.293 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.293 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi053", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/91864548' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/91864548' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:25:54.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi043:172.21.15.43=a;smithi043:[v2:172.21.15.43:3301,v1:172.21.15.43:6790]=c;smithi053:172.21.15.53=b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Saving service mon spec with placement smithi043:172.21.15.43=a;smithi043:[v2:172.21.15.43:3301,v1:172.21.15.43:6790]=c;smithi053:172.21.15.53=b;count:3 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:54.295 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: Deploying daemon mon.c on smithi043 2024-05-15T15:25:54.296 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:54.296 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/50681785' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-15T15:25:54.296 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-05-15T15:25:54.296 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 ceph-mon[29127]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-05-15T15:25:54.585 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 bash[29108]: 44c487fc3e4119314217a291c2701b77ad9418806db84eec9194faf234830d20 2024-05-15T15:25:54.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:54 smithi043 systemd[1]: Started Ceph mon.c for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:25:54.767 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-15T15:25:54.767 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph mon dump -f json 2024-05-15T15:25:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:59.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:25:59.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: mon.a calling monitor election 2024-05-15T15:25:59.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:59.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: mon.c calling monitor election 2024-05-15T15:25:59.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: monmap e2: 2 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: fsmap 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: mgrmap e13: y(active, since 35s) 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: overall HEALTH_OK 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:25:59.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:59.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:59.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:25:59.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: mon.a calling monitor election 2024-05-15T15:25:59.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:59.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: mon.c calling monitor election 2024-05-15T15:25:59.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:25:59.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:25:59.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-15T15:25:59.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: monmap e2: 2 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:25:59.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: fsmap 2024-05-15T15:25:59.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:25:59.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: mgrmap e13: y(active, since 35s) 2024-05-15T15:25:59.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: overall HEALTH_OK 2024-05-15T15:25:59.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:25:59.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:25:59.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:25:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:25:59.724 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-05-15T15:25:59.724 INFO:teuthology.orchestra.run.smithi053.stdout:{"epoch":2,"fsid":"22d82864-12cf-11ef-bc98-c7b262605968","modified":"2024-05-15T15:25:54.248174Z","created":"2024-05-15T15:24:25.819270Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:3300","nonce":0},{"type":"v1","addr":"172.21.15.43:6789","nonce":0}]},"addr":"172.21.15.43:6789/0","public_addr":"172.21.15.43:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:3301","nonce":0},{"type":"v1","addr":"172.21.15.43:6790","nonce":0}]},"addr":"172.21.15.43:6790/0","public_addr":"172.21.15.43:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-15T15:25:59.726 INFO:teuthology.orchestra.run.smithi053.stderr:dumped monmap epoch 2 2024-05-15T15:26:00.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:00 smithi043 ceph-mon[24430]: Deploying daemon mon.b on smithi053 2024-05-15T15:26:00.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:00 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/4233397018' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-15T15:26:00.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:00 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:00.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:00 smithi043 ceph-mon[29127]: Deploying daemon mon.b on smithi053 2024-05-15T15:26:00.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:00 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/4233397018' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-15T15:26:00.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:00 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:01 smithi043 ceph-mon[24430]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:01.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:01 smithi043 ceph-mon[29127]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:02.620 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-15T15:26:02.620 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph mon dump -f json 2024-05-15T15:26:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:03 smithi043 ceph-mon[24430]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:03.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:03 smithi043 ceph-mon[29127]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:03.799 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.preserve_deletes: 0 2024-05-15T15:26:03.799 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.two_write_queues: 0 2024-05-15T15:26:03.799 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.manual_wal_flush: 0 2024-05-15T15:26:03.799 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.atomic_flush: 0 2024-05-15T15:26:03.799 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-15T15:26:03.799 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-15T15:26:03.799 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-15T15:26:03.800 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.log_readahead_size: 0 2024-05-15T15:26:03.800 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-15T15:26:03.800 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.best_efforts_recovery: 0 2024-05-15T15:26:03.800 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-15T15:26:03.800 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-15T15:26:03.800 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.allow_data_in_errors: 0 2024-05-15T15:26:03.800 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.db_host_id: __hostname__ 2024-05-15T15:26:03.800 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_background_jobs: 2 2024-05-15T15:26:03.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_background_compactions: -1 2024-05-15T15:26:03.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_subcompactions: 1 2024-05-15T15:26:03.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-15T15:26:03.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-15T15:26:03.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-15T15:26:03.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_total_wal_size: 0 2024-05-15T15:26:03.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-15T15:26:03.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-15T15:26:03.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-15T15:26:03.802 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-15T15:26:04.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_open_files: -1 2024-05-15T15:26:04.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bytes_per_sync: 0 2024-05-15T15:26:04.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-15T15:26:04.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-15T15:26:04.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_readahead_size: 0 2024-05-15T15:26:04.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_background_flushes: -1 2024-05-15T15:26:04.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Compression algorithms supported: 2024-05-15T15:26:04.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-15T15:26:04.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: kZSTD supported: 0 2024-05-15T15:26:04.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: kXpressCompression supported: 0 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: kLZ4HCCompression supported: 1 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: kLZ4Compression supported: 1 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: kBZip2Compression supported: 0 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: kZlibCompression supported: 1 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: kSnappyCompression supported: 1 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.merge_operator: 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_filter: None 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_filter_factory: None 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.sst_partitioner_factory: None 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-15T15:26:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e205399d00) 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: cache_index_and_filter_blocks: 1 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: pin_top_level_index_and_filter: 1 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: index_type: 0 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: data_block_index_type: 0 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: index_shortening: 1 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: hash_index_allow_collision: 1 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: checksum: 1 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: no_block_cache: 0 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_cache: 0x55e205404170 2024-05-15T15:26:04.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_cache_name: BinnedLRUCache 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_cache_options: 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: capacity : 536870912 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: num_shard_bits : 4 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: strict_capacity_limit : 0 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: high_pri_pool_ratio: 0.000 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_cache_compressed: (nil) 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: persistent_cache: (nil) 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_size: 4096 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_size_deviation: 10 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_restart_interval: 16 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: index_block_restart_interval: 1 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: metadata_block_size: 4096 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: partition_filters: 0 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_delta_encoding: 1 2024-05-15T15:26:04.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: whole_key_filtering: 1 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: verify_compression: 0 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: read_amp_bytes_per_bit: 0 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: format_version: 4 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: enable_index_compression: 1 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_align: 0 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.write_buffer_size: 33554432 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_write_buffer_number: 2 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compression: NoCompression 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bottommost_compression: Disabled 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.prefix_extractor: nullptr 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.num_levels: 7 2024-05-15T15:26:04.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compression_opts.level: 32767 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compression_opts.strategy: 0 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:26:04.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compression_opts.enabled: false 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.target_file_size_base: 67108864 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-15T15:26:04.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.arena_block_size: 4194304 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.disable_auto_compactions: 0 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-15T15:26:04.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.table_properties_collectors: 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.inplace_update_support: 0 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.bloom_locality: 0 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.max_successive_merges: 0 2024-05-15T15:26:04.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-15T15:26:04.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.paranoid_file_checks: 0 2024-05-15T15:26:04.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.force_consistency_checks: 1 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.report_bg_io_stats: 0 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.ttl: 2592000 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.enable_blob_files: false 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.min_blob_size: 0 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.blob_file_size: 268435456 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715786763795931, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-05-15T15:26:04.141 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-05-15T15:26:04.142 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-15T15:26:04.142 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715786763796357, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1715786763, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "39058f3a-6eb1-46a9-9f2d-d7f7ebb93c16", "db_session_id": "SI0GSZS78KQCCYSE3NLD"}} 2024-05-15T15:26:04.142 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-05-15T15:26:04.142 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715786763796835, "job": 1, "event": "recovery_finished"} 2024-05-15T15:26:04.142 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-15T15:26:04.142 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55e2053eaa80 2024-05-15T15:26:04.142 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: DB pointer 0x55e2053fa000 2024-05-15T15:26:04.142 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-15T15:26:04.142 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-15T15:26:04.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** DB Stats ** 2024-05-15T15:26:04.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:26:04.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-15T15:26:04.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-15T15:26:04.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:26:04.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-15T15:26:04.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-15T15:26:04.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:26:04.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:26:04.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** Compaction Stats [default] ** 2024-05-15T15:26:04.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** Compaction Stats [default] ** 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-15T15:26:04.144 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: Cumulative compaction: 0.00 GB write, 0.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: Interval compaction: 0.00 GB write, 0.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** Compaction Stats [default] ** 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:26:04.145 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** Compaction Stats [default] ** 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: Cumulative compaction: 0.00 GB write, 0.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:26:04.146 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: using public_addr v2:172.21.15.53:0/0 -> [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: starting mon.b rank -1 at public addrs [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] at bind addrs [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(???) e0 preinit fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).mds e1 new map 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).mds e1 print_map 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout: e1 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout: legacy client fscid: -1 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout: No filesystems configured 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-05-15T15:26:04.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.a calling monitor election 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:04.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.c calling monitor election 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: monmap e2: 2 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: fsmap 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mgrmap e13: y(active, since 35s) 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: overall HEALTH_OK 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:04.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: Deploying daemon mon.b on smithi053 2024-05-15T15:26:04.150 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/4233397018' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-15T15:26:04.150 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:04.150 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:04.150 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:04.150 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-05-15T15:26:04.150 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 ceph-mon[25205]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-05-15T15:26:04.150 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 bash[25179]: 005e4f2fbae38b90445cfe84d372a44f6237e30370b33d9d5e0056f4a1a55e85 2024-05-15T15:26:04.150 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:03 smithi053 systemd[1]: Started Ceph mon.b for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:26:09.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: mon.a calling monitor election 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: mon.c calling monitor election 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: monmap e3: 3 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],b=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: fsmap 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: mgrmap e13: y(active, since 45s) 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: mon.b (rank 2) addr [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] is down (out of quorum) 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: mon.a calling monitor election 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: mon.c calling monitor election 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.339 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: monmap e3: 3 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],b=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: fsmap 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: mgrmap e13: y(active, since 45s) 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: mon.b (rank 2) addr [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] is down (out of quorum) 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:09.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:09.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:09.437 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-05-15T15:26:09.437 INFO:teuthology.orchestra.run.smithi053.stdout:{"epoch":3,"fsid":"22d82864-12cf-11ef-bc98-c7b262605968","modified":"2024-05-15T15:26:03.818101Z","created":"2024-05-15T15:24:25.819270Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:3300","nonce":0},{"type":"v1","addr":"172.21.15.43:6789","nonce":0}]},"addr":"172.21.15.43:6789/0","public_addr":"172.21.15.43:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:3301","nonce":0},{"type":"v1","addr":"172.21.15.43:6790","nonce":0}]},"addr":"172.21.15.43:6790/0","public_addr":"172.21.15.43:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-15T15:26:09.440 INFO:teuthology.orchestra.run.smithi053.stderr:dumped monmap epoch 3 2024-05-15T15:26:10.212 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-05-15T15:26:10.212 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph config generate-minimal-conf 2024-05-15T15:26:10.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/826415750' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/826415750' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:10.571 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:10.572 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:10.572 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:10.572 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:10.572 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:10.572 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:26:10.572 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:26:10.572 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: mon.a calling monitor election 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: mon.c calling monitor election 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:11.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: monmap e3: 3 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],b=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: fsmap 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: mgrmap e13: y(active, since 45s) 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: mon.b (rank 2) addr [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] is down (out of quorum) 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:11.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/826415750' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-15T15:26:11.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:11.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:11.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:11.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:11.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:11.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:11.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:26:11.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:26:11.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: mon.b calling monitor election 2024-05-15T15:26:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: mon.b calling monitor election 2024-05-15T15:26:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: mon.c calling monitor election 2024-05-15T15:26:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: mon.a calling monitor election 2024-05-15T15:26:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:26:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: monmap e3: 3 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],b=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:26:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: fsmap 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: mgrmap e13: y(active, since 47s) 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: Cluster is now healthy 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: overall HEALTH_OK 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: Reconfiguring mon.c (monmap changed)... 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: Reconfiguring daemon mon.c on smithi043 2024-05-15T15:26:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: mon.b calling monitor election 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: mon.b calling monitor election 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: mon.c calling monitor election 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: mon.a calling monitor election 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: monmap e3: 3 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],b=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: fsmap 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: mgrmap e13: y(active, since 47s) 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: Cluster is now healthy 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: overall HEALTH_OK 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:12.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: Reconfiguring mon.c (monmap changed)... 2024-05-15T15:26:12.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:26:12.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:26:12.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:12.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: Reconfiguring daemon mon.c on smithi043 2024-05-15T15:26:12.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: mon.b calling monitor election 2024-05-15T15:26:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: mon.b calling monitor election 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: mon.c calling monitor election 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: mon.a calling monitor election 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: monmap e3: 3 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],b=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: fsmap 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: osdmap e4: 0 total, 0 up, 0 in 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: mgrmap e13: y(active, since 47s) 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: Cluster is now healthy 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: overall HEALTH_OK 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: Reconfiguring mon.c (monmap changed)... 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:26:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:26:12.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:12.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: Reconfiguring daemon mon.c on smithi043 2024-05-15T15:26:12.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:26:13.007 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:12 smithi043 ceph-mon[24430]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:13.008 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:12 smithi043 ceph-mon[29127]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:13.087 INFO:teuthology.orchestra.run.smithi043.stdout:# minimal ceph.conf for 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:26:13.087 INFO:teuthology.orchestra.run.smithi043.stdout:[global] 2024-05-15T15:26:13.087 INFO:teuthology.orchestra.run.smithi043.stdout: fsid = 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:26:13.087 INFO:teuthology.orchestra.run.smithi043.stdout: mon_host = [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] 2024-05-15T15:26:13.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:12 smithi053 ceph-mon[25205]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:14.047 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:13 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3816616634' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:14.047 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:13 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3816616634' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:14.128 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:13 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3816616634' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:14.498 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-05-15T15:26:14.498 DEBUG:teuthology.orchestra.run.smithi043:> set -ex 2024-05-15T15:26:14.499 DEBUG:teuthology.orchestra.run.smithi043:> sudo dd of=/etc/ceph/ceph.conf 2024-05-15T15:26:14.527 DEBUG:teuthology.orchestra.run.smithi043:> set -ex 2024-05-15T15:26:14.527 DEBUG:teuthology.orchestra.run.smithi043:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:26:14.593 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-05-15T15:26:14.593 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd of=/etc/ceph/ceph.conf 2024-05-15T15:26:14.625 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-05-15T15:26:14.626 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:26:14.692 INFO:tasks.cephadm:Adding mgr.y on smithi043 2024-05-15T15:26:14.692 INFO:tasks.cephadm:Adding mgr.x on smithi053 2024-05-15T15:26:14.692 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch apply mgr '2;smithi043=y;smithi053=x' 2024-05-15T15:26:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:15 smithi053 ceph-mon[25205]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:15 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:15 smithi053 ceph-mon[25205]: Reconfiguring mon.b (monmap changed)... 2024-05-15T15:26:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:15 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:26:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:15 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:26:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:15 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:15 smithi053 ceph-mon[25205]: Reconfiguring daemon mon.b on smithi053 2024-05-15T15:26:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[29127]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[29127]: Reconfiguring mon.b (monmap changed)... 2024-05-15T15:26:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:26:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:26:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[29127]: Reconfiguring daemon mon.b on smithi053 2024-05-15T15:26:15.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[24430]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:15.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:15.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[24430]: Reconfiguring mon.b (monmap changed)... 2024-05-15T15:26:15.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:26:15.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:26:15.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:15.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:15 smithi043 ceph-mon[24430]: Reconfiguring daemon mon.b on smithi053 2024-05-15T15:26:16.729 INFO:teuthology.orchestra.run.smithi053.stdout:Scheduled mgr update... 2024-05-15T15:26:17.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:17.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:17.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:17.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:17.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:17.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:17.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:17.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:26:17.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-15T15:26:17.361 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:26:17.361 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:17.362 DEBUG:teuthology.orchestra.run.smithi053:mgr.x> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.x.service 2024-05-15T15:26:17.364 INFO:tasks.cephadm:Deploying OSDs... 2024-05-15T15:26:17.364 DEBUG:teuthology.orchestra.run.smithi043:> set -ex 2024-05-15T15:26:17.364 DEBUG:teuthology.orchestra.run.smithi043:> dd if=/scratch_devs of=/dev/stdout 2024-05-15T15:26:17.384 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-15T15:26:17.385 DEBUG:teuthology.orchestra.run.smithi043:> stat /dev/vg_nvme/lv_1 2024-05-15T15:26:17.444 INFO:teuthology.orchestra.run.smithi043.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-15T15:26:17.444 INFO:teuthology.orchestra.run.smithi043.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-15T15:26:17.444 INFO:teuthology.orchestra.run.smithi043.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-05-15T15:26:17.444 INFO:teuthology.orchestra.run.smithi043.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-15T15:26:17.445 INFO:teuthology.orchestra.run.smithi043.stdout:Context: system_u:object_r:device_t:s0 2024-05-15T15:26:17.445 INFO:teuthology.orchestra.run.smithi043.stdout:Access: 2024-05-15 15:25:53.709167414 +0000 2024-05-15T15:26:17.445 INFO:teuthology.orchestra.run.smithi043.stdout:Modify: 2024-05-15 15:23:11.066412613 +0000 2024-05-15T15:26:17.445 INFO:teuthology.orchestra.run.smithi043.stdout:Change: 2024-05-15 15:23:11.066412613 +0000 2024-05-15T15:26:17.445 INFO:teuthology.orchestra.run.smithi043.stdout: Birth: 2024-05-15 15:23:11.066412613 +0000 2024-05-15T15:26:17.445 DEBUG:teuthology.orchestra.run.smithi043:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-15T15:26:17.512 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:17.512 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:17.512 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:17.512 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:17.512 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:17.512 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:17.513 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:17.513 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:26:17.513 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-15T15:26:17.513 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:26:17.513 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:17.513 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:17.513 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:17.513 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:17.514 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:17.514 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:17.514 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:17.514 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:17.514 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:26:17.514 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-15T15:26:17.514 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:26:17.514 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:17.515 INFO:teuthology.orchestra.run.smithi043.stderr:1+0 records in 2024-05-15T15:26:17.515 INFO:teuthology.orchestra.run.smithi043.stderr:1+0 records out 2024-05-15T15:26:17.515 INFO:teuthology.orchestra.run.smithi043.stderr:512 bytes copied, 0.000128835 s, 4.0 MB/s 2024-05-15T15:26:17.516 DEBUG:teuthology.orchestra.run.smithi043:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-15T15:26:17.575 DEBUG:teuthology.orchestra.run.smithi043:> stat /dev/vg_nvme/lv_2 2024-05-15T15:26:17.634 INFO:teuthology.orchestra.run.smithi043.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-15T15:26:17.634 INFO:teuthology.orchestra.run.smithi043.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-15T15:26:17.634 INFO:teuthology.orchestra.run.smithi043.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-05-15T15:26:17.634 INFO:teuthology.orchestra.run.smithi043.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-15T15:26:17.634 INFO:teuthology.orchestra.run.smithi043.stdout:Context: system_u:object_r:device_t:s0 2024-05-15T15:26:17.634 INFO:teuthology.orchestra.run.smithi043.stdout:Access: 2024-05-15 15:25:53.709167414 +0000 2024-05-15T15:26:17.634 INFO:teuthology.orchestra.run.smithi043.stdout:Modify: 2024-05-15 15:23:11.411408398 +0000 2024-05-15T15:26:17.634 INFO:teuthology.orchestra.run.smithi043.stdout:Change: 2024-05-15 15:23:11.411408398 +0000 2024-05-15T15:26:17.634 INFO:teuthology.orchestra.run.smithi043.stdout: Birth: 2024-05-15 15:23:11.411408398 +0000 2024-05-15T15:26:17.635 DEBUG:teuthology.orchestra.run.smithi043:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-15T15:26:17.701 INFO:teuthology.orchestra.run.smithi043.stderr:1+0 records in 2024-05-15T15:26:17.702 INFO:teuthology.orchestra.run.smithi043.stderr:1+0 records out 2024-05-15T15:26:17.702 INFO:teuthology.orchestra.run.smithi043.stderr:512 bytes copied, 0.000137092 s, 3.7 MB/s 2024-05-15T15:26:17.703 DEBUG:teuthology.orchestra.run.smithi043:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-15T15:26:17.762 DEBUG:teuthology.orchestra.run.smithi043:> stat /dev/vg_nvme/lv_3 2024-05-15T15:26:17.821 INFO:teuthology.orchestra.run.smithi043.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-15T15:26:17.822 INFO:teuthology.orchestra.run.smithi043.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-15T15:26:17.822 INFO:teuthology.orchestra.run.smithi043.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-05-15T15:26:17.822 INFO:teuthology.orchestra.run.smithi043.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-15T15:26:17.822 INFO:teuthology.orchestra.run.smithi043.stdout:Context: system_u:object_r:device_t:s0 2024-05-15T15:26:17.822 INFO:teuthology.orchestra.run.smithi043.stdout:Access: 2024-05-15 15:25:53.710167402 +0000 2024-05-15T15:26:17.822 INFO:teuthology.orchestra.run.smithi043.stdout:Modify: 2024-05-15 15:23:11.767404049 +0000 2024-05-15T15:26:17.822 INFO:teuthology.orchestra.run.smithi043.stdout:Change: 2024-05-15 15:23:11.767404049 +0000 2024-05-15T15:26:17.822 INFO:teuthology.orchestra.run.smithi043.stdout: Birth: 2024-05-15 15:23:11.767404049 +0000 2024-05-15T15:26:17.822 DEBUG:teuthology.orchestra.run.smithi043:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-15T15:26:17.890 INFO:teuthology.orchestra.run.smithi043.stderr:1+0 records in 2024-05-15T15:26:17.890 INFO:teuthology.orchestra.run.smithi043.stderr:1+0 records out 2024-05-15T15:26:17.890 INFO:teuthology.orchestra.run.smithi043.stderr:512 bytes copied, 0.000114635 s, 4.5 MB/s 2024-05-15T15:26:17.891 DEBUG:teuthology.orchestra.run.smithi043:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-15T15:26:17.951 DEBUG:teuthology.orchestra.run.smithi043:> stat /dev/vg_nvme/lv_4 2024-05-15T15:26:18.010 INFO:teuthology.orchestra.run.smithi043.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-15T15:26:18.010 INFO:teuthology.orchestra.run.smithi043.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-15T15:26:18.010 INFO:teuthology.orchestra.run.smithi043.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-05-15T15:26:18.011 INFO:teuthology.orchestra.run.smithi043.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-15T15:26:18.011 INFO:teuthology.orchestra.run.smithi043.stdout:Context: system_u:object_r:device_t:s0 2024-05-15T15:26:18.011 INFO:teuthology.orchestra.run.smithi043.stdout:Access: 2024-05-15 15:25:53.710167402 +0000 2024-05-15T15:26:18.011 INFO:teuthology.orchestra.run.smithi043.stdout:Modify: 2024-05-15 15:23:12.109399872 +0000 2024-05-15T15:26:18.011 INFO:teuthology.orchestra.run.smithi043.stdout:Change: 2024-05-15 15:23:12.109399872 +0000 2024-05-15T15:26:18.011 INFO:teuthology.orchestra.run.smithi043.stdout: Birth: 2024-05-15 15:23:12.109399872 +0000 2024-05-15T15:26:18.011 DEBUG:teuthology.orchestra.run.smithi043:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-15T15:26:18.078 INFO:teuthology.orchestra.run.smithi043.stderr:1+0 records in 2024-05-15T15:26:18.079 INFO:teuthology.orchestra.run.smithi043.stderr:1+0 records out 2024-05-15T15:26:18.079 INFO:teuthology.orchestra.run.smithi043.stderr:512 bytes copied, 0.000127766 s, 4.0 MB/s 2024-05-15T15:26:18.080 DEBUG:teuthology.orchestra.run.smithi043:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-15T15:26:18.139 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-05-15T15:26:18.139 DEBUG:teuthology.orchestra.run.smithi053:> dd if=/scratch_devs of=/dev/stdout 2024-05-15T15:26:18.157 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-15T15:26:18.158 DEBUG:teuthology.orchestra.run.smithi053:> stat /dev/vg_nvme/lv_1 2024-05-15T15:26:18.222 INFO:teuthology.orchestra.run.smithi053.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-15T15:26:18.222 INFO:teuthology.orchestra.run.smithi053.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-15T15:26:18.223 INFO:teuthology.orchestra.run.smithi053.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-05-15T15:26:18.223 INFO:teuthology.orchestra.run.smithi053.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-15T15:26:18.223 INFO:teuthology.orchestra.run.smithi053.stdout:Context: system_u:object_r:device_t:s0 2024-05-15T15:26:18.223 INFO:teuthology.orchestra.run.smithi053.stdout:Access: 2024-05-15 15:26:02.820407965 +0000 2024-05-15T15:26:18.223 INFO:teuthology.orchestra.run.smithi053.stdout:Modify: 2024-05-15 15:23:11.049444795 +0000 2024-05-15T15:26:18.223 INFO:teuthology.orchestra.run.smithi053.stdout:Change: 2024-05-15 15:23:11.049444795 +0000 2024-05-15T15:26:18.223 INFO:teuthology.orchestra.run.smithi053.stdout: Birth: 2024-05-15 15:23:11.049444795 +0000 2024-05-15T15:26:18.224 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-15T15:26:18.302 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records in 2024-05-15T15:26:18.302 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records out 2024-05-15T15:26:18.302 INFO:teuthology.orchestra.run.smithi053.stderr:512 bytes copied, 0.000114841 s, 4.5 MB/s 2024-05-15T15:26:18.304 DEBUG:teuthology.orchestra.run.smithi053:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-15T15:26:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:18 smithi043 ceph-mon[24430]: from='client.24107 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi043=y;smithi053=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:26:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:18 smithi043 ceph-mon[24430]: Saving service mgr spec with placement smithi043=y;smithi053=x;count:2 2024-05-15T15:26:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:18 smithi043 ceph-mon[24430]: Deploying daemon mgr.x on smithi053 2024-05-15T15:26:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:18 smithi043 ceph-mon[29127]: from='client.24107 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi043=y;smithi053=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:26:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:18 smithi043 ceph-mon[29127]: Saving service mgr spec with placement smithi043=y;smithi053=x;count:2 2024-05-15T15:26:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:18 smithi043 ceph-mon[29127]: Deploying daemon mgr.x on smithi053 2024-05-15T15:26:18.375 DEBUG:teuthology.orchestra.run.smithi053:> stat /dev/vg_nvme/lv_2 2024-05-15T15:26:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:18 smithi053 ceph-mon[25205]: from='client.24107 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi043=y;smithi053=x", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:26:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:18 smithi053 ceph-mon[25205]: Saving service mgr spec with placement smithi043=y;smithi053=x;count:2 2024-05-15T15:26:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:18 smithi053 ceph-mon[25205]: Deploying daemon mgr.x on smithi053 2024-05-15T15:26:18.400 INFO:teuthology.orchestra.run.smithi053.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-15T15:26:18.400 INFO:teuthology.orchestra.run.smithi053.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-15T15:26:18.400 INFO:teuthology.orchestra.run.smithi053.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-05-15T15:26:18.400 INFO:teuthology.orchestra.run.smithi053.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-15T15:26:18.400 INFO:teuthology.orchestra.run.smithi053.stdout:Context: system_u:object_r:device_t:s0 2024-05-15T15:26:18.400 INFO:teuthology.orchestra.run.smithi053.stdout:Access: 2024-05-15 15:26:02.820407965 +0000 2024-05-15T15:26:18.400 INFO:teuthology.orchestra.run.smithi053.stdout:Modify: 2024-05-15 15:23:11.394438021 +0000 2024-05-15T15:26:18.400 INFO:teuthology.orchestra.run.smithi053.stdout:Change: 2024-05-15 15:23:11.394438021 +0000 2024-05-15T15:26:18.400 INFO:teuthology.orchestra.run.smithi053.stdout: Birth: 2024-05-15 15:23:11.394438021 +0000 2024-05-15T15:26:18.401 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-15T15:26:18.470 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records in 2024-05-15T15:26:18.470 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records out 2024-05-15T15:26:18.470 INFO:teuthology.orchestra.run.smithi053.stderr:512 bytes copied, 0.000108302 s, 4.7 MB/s 2024-05-15T15:26:18.471 DEBUG:teuthology.orchestra.run.smithi053:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-15T15:26:18.536 DEBUG:teuthology.orchestra.run.smithi053:> stat /dev/vg_nvme/lv_3 2024-05-15T15:26:18.598 INFO:teuthology.orchestra.run.smithi053.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-15T15:26:18.598 INFO:teuthology.orchestra.run.smithi053.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-15T15:26:18.598 INFO:teuthology.orchestra.run.smithi053.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-05-15T15:26:18.598 INFO:teuthology.orchestra.run.smithi053.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-15T15:26:18.599 INFO:teuthology.orchestra.run.smithi053.stdout:Context: system_u:object_r:device_t:s0 2024-05-15T15:26:18.599 INFO:teuthology.orchestra.run.smithi053.stdout:Access: 2024-05-15 15:26:02.820407965 +0000 2024-05-15T15:26:18.599 INFO:teuthology.orchestra.run.smithi053.stdout:Modify: 2024-05-15 15:23:11.735431326 +0000 2024-05-15T15:26:18.599 INFO:teuthology.orchestra.run.smithi053.stdout:Change: 2024-05-15 15:23:11.735431326 +0000 2024-05-15T15:26:18.599 INFO:teuthology.orchestra.run.smithi053.stdout: Birth: 2024-05-15 15:23:11.735431326 +0000 2024-05-15T15:26:18.599 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-15T15:26:18.684 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records in 2024-05-15T15:26:18.684 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records out 2024-05-15T15:26:18.684 INFO:teuthology.orchestra.run.smithi053.stderr:512 bytes copied, 0.000109999 s, 4.7 MB/s 2024-05-15T15:26:18.686 DEBUG:teuthology.orchestra.run.smithi053:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-15T15:26:18.756 DEBUG:teuthology.orchestra.run.smithi053:> stat /dev/vg_nvme/lv_4 2024-05-15T15:26:18.818 INFO:teuthology.orchestra.run.smithi053.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-15T15:26:18.819 INFO:teuthology.orchestra.run.smithi053.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-15T15:26:18.819 INFO:teuthology.orchestra.run.smithi053.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-05-15T15:26:18.819 INFO:teuthology.orchestra.run.smithi053.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-15T15:26:18.819 INFO:teuthology.orchestra.run.smithi053.stdout:Context: system_u:object_r:device_t:s0 2024-05-15T15:26:18.819 INFO:teuthology.orchestra.run.smithi053.stdout:Access: 2024-05-15 15:26:18.634098970 +0000 2024-05-15T15:26:18.819 INFO:teuthology.orchestra.run.smithi053.stdout:Modify: 2024-05-15 15:23:12.071424729 +0000 2024-05-15T15:26:18.819 INFO:teuthology.orchestra.run.smithi053.stdout:Change: 2024-05-15 15:23:12.071424729 +0000 2024-05-15T15:26:18.819 INFO:teuthology.orchestra.run.smithi053.stdout: Birth: 2024-05-15 15:23:12.071424729 +0000 2024-05-15T15:26:18.819 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-15T15:26:18.892 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records in 2024-05-15T15:26:18.892 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records out 2024-05-15T15:26:18.892 INFO:teuthology.orchestra.run.smithi053.stderr:512 bytes copied, 0.000100021 s, 5.1 MB/s 2024-05-15T15:26:18.893 DEBUG:teuthology.orchestra.run.smithi053:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-15T15:26:18.958 INFO:tasks.cephadm:Deploying osd.0 on smithi043 with /dev/vg_nvme/lv_4... 2024-05-15T15:26:18.958 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-15T15:26:19.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:19 smithi053 ceph-mon[25205]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:19.285 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:19 smithi043 ceph-mon[24430]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:19.285 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:19 smithi043 ceph-mon[29127]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:19.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:19 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:19.697+0000 7f577c840000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:26:20.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:20 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:20.041+0000 7f577c840000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:26:20.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:20 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:20.130+0000 7f577c840000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:26:20.565 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:20 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:20.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:20 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:20.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:20 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:20.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:20 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:20.566 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:20 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:20.566 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:20 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:20.566 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:20 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:20.566 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:20 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:20 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:20 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:20 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:20 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:21.130 INFO:teuthology.orchestra.run.smithi043.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-15T15:26:21.130 INFO:teuthology.orchestra.run.smithi043.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-15T15:26:21.130 INFO:teuthology.orchestra.run.smithi043.stderr: stderr: 10+0 records in 2024-05-15T15:26:21.131 INFO:teuthology.orchestra.run.smithi043.stderr:10+0 records out 2024-05-15T15:26:21.131 INFO:teuthology.orchestra.run.smithi043.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00838722 s, 1.3 GB/s 2024-05-15T15:26:21.131 INFO:teuthology.orchestra.run.smithi043.stderr:--> Zapping successful for: 2024-05-15T15:26:21.596 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:21 smithi053 ceph-mon[25205]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:21.596 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:21 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:21.596 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:21 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:21.596 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:21 smithi053 ceph-mon[25205]: Reconfiguring mgr.y (unknown last config time)... 2024-05-15T15:26:21.596 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:21 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:26:21.596 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:21 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:26:21.596 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:21 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:21.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:21 smithi053 ceph-mon[25205]: Reconfiguring daemon mgr.y on smithi043 2024-05-15T15:26:21.597 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:21 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:21.339+0000 7f577c840000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:26:21.597 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:21 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:21.509+0000 7f577c840000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:26:21.597 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:21 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:21.594+0000 7f577c840000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:26:21.808 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[24430]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:21.808 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:21.808 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:21.808 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[24430]: Reconfiguring mgr.y (unknown last config time)... 2024-05-15T15:26:21.808 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:26:21.808 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:26:21.809 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:21.809 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[24430]: Reconfiguring daemon mgr.y on smithi043 2024-05-15T15:26:21.809 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[29127]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:21.809 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:21.809 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:21.809 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[29127]: Reconfiguring mgr.y (unknown last config time)... 2024-05-15T15:26:21.809 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:26:21.809 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:26:21.809 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:21.810 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:21 smithi043 ceph-mon[29127]: Reconfiguring daemon mgr.y on smithi043 2024-05-15T15:26:22.360 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:21 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:21.918+0000 7f577c840000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:26:22.360 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:22 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:22.012+0000 7f577c840000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:26:22.360 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:22 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:22.108+0000 7f577c840000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:26:22.522 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch daemon add osd smithi043:vg_nvme/lv_4 2024-05-15T15:26:22.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:22 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:22.358+0000 7f577c840000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:26:23.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:22 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:22.819+0000 7f577c840000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:26:23.428 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:23.329+0000 7f577c840000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:26:23.428 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:23 smithi053 ceph-mon[25205]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:23.428 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:23.428 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:23.428 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:23.428 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:23.429 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:23.484 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[24430]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:23.485 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:23.485 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:23.485 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:23.485 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:23.485 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:23.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[29127]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:23.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:23.486 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:23.486 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:23.486 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:23.486 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:23.735 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:23.426+0000 7f577c840000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:26:23.735 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:23.732+0000 7f577c840000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:26:24.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:23.824+0000 7f577c840000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:26:24.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:26:24.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:26:24.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:24.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:24 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:26:24.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:24 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:26:24.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:24 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:24.481 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:24 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:26:24.482 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:24 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:26:24.482 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:24 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:24.482 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:24.245+0000 7f577c840000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:26:24.482 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:24.382+0000 7f577c840000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:26:24.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:24.478+0000 7f577c840000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:26:24.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:24.596+0000 7f577c840000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:26:24.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:24.687+0000 7f577c840000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:26:25.484 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:25 smithi043 ceph-mon[24430]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:25.485 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:25 smithi043 ceph-mon[24430]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:26:25.485 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:26:25.485 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:26:25.485 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:25.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:25 smithi043 ceph-mon[29127]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:25.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:25 smithi043 ceph-mon[29127]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:26:25.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:25 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:26:25.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:25 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:26:25.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:25 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:25.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:25.260+0000 7f577c840000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:26:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:25 smithi053 ceph-mon[25205]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:25 smithi053 ceph-mon[25205]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:26:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:25 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:26:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:25 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:26:25.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:25 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:26.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:26:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:26:25.664+0000 7f577c840000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:26:26.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:26 smithi043 ceph-mon[29127]: Standby manager daemon x started 2024-05-15T15:26:26.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:26 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:26:26.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:26 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:26:26.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:26 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:26:26.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:26 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:26:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:26 smithi043 ceph-mon[24430]: Standby manager daemon x started 2024-05-15T15:26:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:26 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:26:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:26 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:26:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:26 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:26:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:26 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:26:26.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:26 smithi053 ceph-mon[25205]: Standby manager daemon x started 2024-05-15T15:26:26.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:26 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:26:26.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:26 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:26:26.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:26 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:26:26.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:26 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/3969889811' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:26:27.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[24430]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:27.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[24430]: mgrmap e14: y(active, since 62s), standbys: x 2024-05-15T15:26:27.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:26:27.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2253597097' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ac451f1-36bd-4ff8-8b90-92a83e712159"}]: dispatch 2024-05-15T15:26:27.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2253597097' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7ac451f1-36bd-4ff8-8b90-92a83e712159"}]': finished 2024-05-15T15:26:27.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[24430]: osdmap e5: 1 total, 0 up, 1 in 2024-05-15T15:26:27.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[29127]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[29127]: mgrmap e14: y(active, since 62s), standbys: x 2024-05-15T15:26:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:26:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2253597097' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ac451f1-36bd-4ff8-8b90-92a83e712159"}]: dispatch 2024-05-15T15:26:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2253597097' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7ac451f1-36bd-4ff8-8b90-92a83e712159"}]': finished 2024-05-15T15:26:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[29127]: osdmap e5: 1 total, 0 up, 1 in 2024-05-15T15:26:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:27 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:27 smithi053 ceph-mon[25205]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:27 smithi053 ceph-mon[25205]: mgrmap e14: y(active, since 62s), standbys: x 2024-05-15T15:26:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:27 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:26:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:27 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2253597097' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ac451f1-36bd-4ff8-8b90-92a83e712159"}]: dispatch 2024-05-15T15:26:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:27 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2253597097' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7ac451f1-36bd-4ff8-8b90-92a83e712159"}]': finished 2024-05-15T15:26:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:27 smithi053 ceph-mon[25205]: osdmap e5: 1 total, 0 up, 1 in 2024-05-15T15:26:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:27 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:28.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:28 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3455771975' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:26:28.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:28 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3455771975' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:26:28.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:28 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3455771975' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:26:29.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:29 smithi043 ceph-mon[24430]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:29.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:29 smithi043 ceph-mon[29127]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:29.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:29 smithi053 ceph-mon[25205]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:30.996 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:30 smithi043 ceph-mon[24430]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:30.996 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:30 smithi043 ceph-mon[29127]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:30 smithi053 ceph-mon[25205]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:33.316 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:32 smithi043 ceph-mon[24430]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:33.316 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:32 smithi043 ceph-mon[29127]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:33.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:32 smithi053 ceph-mon[25205]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:34.162 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-15T15:26:34.162 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:34.162 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:33 smithi043 ceph-mon[24430]: Deploying daemon osd.0 on smithi043 2024-05-15T15:26:34.162 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-15T15:26:34.162 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:34.162 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:33 smithi043 ceph-mon[29127]: Deploying daemon osd.0 on smithi043 2024-05-15T15:26:34.281 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-15T15:26:34.282 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:34.282 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:33 smithi053 ceph-mon[25205]: Deploying daemon osd.0 on smithi043 2024-05-15T15:26:35.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:34 smithi043 ceph-mon[29127]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:35.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:34 smithi043 ceph-mon[24430]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:34 smithi053 ceph-mon[25205]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:36.973 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:36 smithi043 ceph-mon[24430]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:36.974 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:36 smithi043 ceph-mon[29127]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:37.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:36 smithi053 ceph-mon[25205]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:39.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:38 smithi053 ceph-mon[25205]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:39.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:38 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:39.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:38 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:39.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:38 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:39.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:38 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:39.193 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:38 smithi043 ceph-mon[24430]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:39.193 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:38 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:39.193 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:38 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:39.193 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:38 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:39.193 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:38 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:39.193 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:38 smithi043 ceph-mon[29127]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:39.193 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:38 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:39.193 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:38 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:39.194 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:38 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:39.194 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:38 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:40 smithi043 ceph-mon[24430]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:40 smithi043 ceph-mon[29127]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:40.968 INFO:teuthology.orchestra.run.smithi043.stdout:Created osd(s) 0 on host 'smithi043' 2024-05-15T15:26:41.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:40 smithi053 ceph-mon[25205]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:41.593 DEBUG:teuthology.orchestra.run.smithi043:osd.0> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.0.service 2024-05-15T15:26:41.596 INFO:tasks.cephadm:Deploying osd.1 on smithi043 with /dev/vg_nvme/lv_3... 2024-05-15T15:26:41.596 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-15T15:26:42.257 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:41 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:42.258 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:41 smithi043 ceph-mon[24430]: from='osd.0 [v2:172.21.15.43:6802/2576871983,v1:172.21.15.43:6803/2576871983]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:26:42.258 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:41 smithi043 ceph-mon[24430]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:26:42.258 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:41 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:42.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:41 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:42.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:41 smithi043 ceph-mon[29127]: from='osd.0 [v2:172.21.15.43:6802/2576871983,v1:172.21.15.43:6803/2576871983]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:26:42.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:41 smithi043 ceph-mon[29127]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:26:42.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:41 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:42.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:41 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:42.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:41 smithi053 ceph-mon[25205]: from='osd.0 [v2:172.21.15.43:6802/2576871983,v1:172.21.15.43:6803/2576871983]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:26:42.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:41 smithi053 ceph-mon[25205]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:26:42.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:41 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:43.032 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[24430]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:43.032 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[24430]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-15T15:26:43.032 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[24430]: osdmap e6: 1 total, 0 up, 1 in 2024-05-15T15:26:43.033 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:43.033 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[24430]: from='osd.0 [v2:172.21.15.43:6802/2576871983,v1:172.21.15.43:6803/2576871983]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:26:43.033 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[24430]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:26:43.033 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[29127]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:43.033 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[29127]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-15T15:26:43.033 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[29127]: osdmap e6: 1 total, 0 up, 1 in 2024-05-15T15:26:43.033 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:43.033 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[29127]: from='osd.0 [v2:172.21.15.43:6802/2576871983,v1:172.21.15.43:6803/2576871983]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:26:43.034 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:42 smithi043 ceph-mon[29127]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:26:43.034 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:26:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[31968]: 2024-05-15T15:26:42.975+0000 7fae83dbc700 -1 osd.0 0 waiting for initial osdmap 2024-05-15T15:26:43.034 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:26:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[31968]: 2024-05-15T15:26:42.978+0000 7fae7cf50700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:26:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:42 smithi053 ceph-mon[25205]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:42 smithi053 ceph-mon[25205]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-15T15:26:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:42 smithi053 ceph-mon[25205]: osdmap e6: 1 total, 0 up, 1 in 2024-05-15T15:26:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:42 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:42 smithi053 ceph-mon[25205]: from='osd.0 [v2:172.21.15.43:6802/2576871983,v1:172.21.15.43:6803/2576871983]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:26:43.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:42 smithi053 ceph-mon[25205]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:26:43.518 INFO:teuthology.orchestra.run.smithi043.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-15T15:26:43.518 INFO:teuthology.orchestra.run.smithi043.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-15T15:26:43.518 INFO:teuthology.orchestra.run.smithi043.stderr: stderr: 10+0 records in 2024-05-15T15:26:43.518 INFO:teuthology.orchestra.run.smithi043.stderr:10+0 records out 2024-05-15T15:26:43.518 INFO:teuthology.orchestra.run.smithi043.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00942955 s, 1.1 GB/s 2024-05-15T15:26:43.518 INFO:teuthology.orchestra.run.smithi043.stderr:--> Zapping successful for: 2024-05-15T15:26:44.194 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch daemon add osd smithi043:vg_nvme/lv_3 2024-05-15T15:26:44.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:43 smithi043 ceph-mon[29127]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:26:44.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:43 smithi043 ceph-mon[29127]: osdmap e7: 1 total, 0 up, 1 in 2024-05-15T15:26:44.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:44.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:44.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:43 smithi043 ceph-mon[24430]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:26:44.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:43 smithi043 ceph-mon[24430]: osdmap e7: 1 total, 0 up, 1 in 2024-05-15T15:26:44.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:44.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:43 smithi053 ceph-mon[25205]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:26:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:43 smithi053 ceph-mon[25205]: osdmap e7: 1 total, 0 up, 1 in 2024-05-15T15:26:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:45.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[24430]: purged_snaps scrub starts 2024-05-15T15:26:45.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[24430]: purged_snaps scrub ok 2024-05-15T15:26:45.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[24430]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:45.212 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[24430]: osd.0 [v2:172.21.15.43:6802/2576871983,v1:172.21.15.43:6803/2576871983] boot 2024-05-15T15:26:45.212 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[24430]: osdmap e8: 1 total, 1 up, 1 in 2024-05-15T15:26:45.212 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:45.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[29127]: purged_snaps scrub starts 2024-05-15T15:26:45.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[29127]: purged_snaps scrub ok 2024-05-15T15:26:45.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[29127]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:45.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[29127]: osd.0 [v2:172.21.15.43:6802/2576871983,v1:172.21.15.43:6803/2576871983] boot 2024-05-15T15:26:45.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[29127]: osdmap e8: 1 total, 1 up, 1 in 2024-05-15T15:26:45.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:44 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:44 smithi053 ceph-mon[25205]: purged_snaps scrub starts 2024-05-15T15:26:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:44 smithi053 ceph-mon[25205]: purged_snaps scrub ok 2024-05-15T15:26:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:44 smithi053 ceph-mon[25205]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-15T15:26:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:44 smithi053 ceph-mon[25205]: osd.0 [v2:172.21.15.43:6802/2576871983,v1:172.21.15.43:6803/2576871983] boot 2024-05-15T15:26:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:44 smithi053 ceph-mon[25205]: osdmap e8: 1 total, 1 up, 1 in 2024-05-15T15:26:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:44 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:26:47.303 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[29127]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:47.303 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[29127]: osdmap e9: 1 total, 1 up, 1 in 2024-05-15T15:26:47.304 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.304 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:26:47.304 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.304 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.304 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:47.304 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:47.304 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:47.304 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.304 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[24430]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:47.304 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[24430]: osdmap e9: 1 total, 1 up, 1 in 2024-05-15T15:26:47.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:26:47.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:47.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:47.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:47.305 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:46 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:46 smithi053 ceph-mon[25205]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:47.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:46 smithi053 ceph-mon[25205]: osdmap e9: 1 total, 1 up, 1 in 2024-05-15T15:26:47.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:46 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:46 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:26:47.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:46 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:46 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:47.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:46 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:26:47.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:46 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:47.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:46 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:26:47.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:46 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:26:48.242 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:47 smithi043 ceph-mon[24430]: Detected new or changed devices on smithi043 2024-05-15T15:26:48.242 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:47 smithi043 ceph-mon[24430]: Adjusting osd_memory_target on smithi043 to 16069M 2024-05-15T15:26:48.243 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:47 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:26:48.243 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:47 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:26:48.243 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:47 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:48.243 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:47 smithi043 ceph-mon[29127]: Detected new or changed devices on smithi043 2024-05-15T15:26:48.243 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:47 smithi043 ceph-mon[29127]: Adjusting osd_memory_target on smithi043 to 16069M 2024-05-15T15:26:48.243 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:47 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:26:48.243 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:47 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:26:48.243 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:47 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:48.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:47 smithi053 ceph-mon[25205]: Detected new or changed devices on smithi043 2024-05-15T15:26:48.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:47 smithi053 ceph-mon[25205]: Adjusting osd_memory_target on smithi043 to 16069M 2024-05-15T15:26:48.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:47 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:26:48.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:47 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:26:48.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:47 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:49.201 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:48 smithi043 ceph-mon[24430]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:26:49.201 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:48 smithi043 ceph-mon[24430]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:49.202 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:48 smithi043 ceph-mon[29127]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:26:49.202 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:48 smithi043 ceph-mon[29127]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:49.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:48 smithi053 ceph-mon[25205]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:26:49.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:48 smithi053 ceph-mon[25205]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:51.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[24430]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:51.408 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/383936900' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c35bd4a6-ea35-419f-a5cc-fc97f9eb509d"}]: dispatch 2024-05-15T15:26:51.408 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c35bd4a6-ea35-419f-a5cc-fc97f9eb509d"}]: dispatch 2024-05-15T15:26:51.408 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c35bd4a6-ea35-419f-a5cc-fc97f9eb509d"}]': finished 2024-05-15T15:26:51.408 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[24430]: osdmap e10: 2 total, 1 up, 2 in 2024-05-15T15:26:51.408 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:26:51.408 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1314889742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:26:51.408 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[29127]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:51.408 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/383936900' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c35bd4a6-ea35-419f-a5cc-fc97f9eb509d"}]: dispatch 2024-05-15T15:26:51.408 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c35bd4a6-ea35-419f-a5cc-fc97f9eb509d"}]: dispatch 2024-05-15T15:26:51.409 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c35bd4a6-ea35-419f-a5cc-fc97f9eb509d"}]': finished 2024-05-15T15:26:51.409 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[29127]: osdmap e10: 2 total, 1 up, 2 in 2024-05-15T15:26:51.409 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:26:51.409 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:50 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1314889742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:26:51.409 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:50 smithi053 ceph-mon[25205]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:51.409 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:50 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/383936900' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c35bd4a6-ea35-419f-a5cc-fc97f9eb509d"}]: dispatch 2024-05-15T15:26:51.409 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:50 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c35bd4a6-ea35-419f-a5cc-fc97f9eb509d"}]: dispatch 2024-05-15T15:26:51.409 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:50 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c35bd4a6-ea35-419f-a5cc-fc97f9eb509d"}]': finished 2024-05-15T15:26:51.409 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:50 smithi053 ceph-mon[25205]: osdmap e10: 2 total, 1 up, 2 in 2024-05-15T15:26:51.410 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:50 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:26:51.410 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:50 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1314889742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:26:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:52 smithi043 ceph-mon[24430]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:53.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:52 smithi043 ceph-mon[29127]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:53.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:52 smithi053 ceph-mon[25205]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:55.149 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:54 smithi043 ceph-mon[24430]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:55.150 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:54 smithi043 ceph-mon[29127]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:55.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:54 smithi053 ceph-mon[25205]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:56 smithi043 ceph-mon[24430]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:56 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-15T15:26:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:56 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:56 smithi043 ceph-mon[24430]: Deploying daemon osd.1 on smithi043 2024-05-15T15:26:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:56 smithi043 ceph-mon[29127]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:56 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-15T15:26:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:56 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:57.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:56 smithi043 ceph-mon[29127]: Deploying daemon osd.1 on smithi043 2024-05-15T15:26:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:56 smithi053 ceph-mon[25205]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:56 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-15T15:26:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:56 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:26:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:56 smithi053 ceph-mon[25205]: Deploying daemon osd.1 on smithi043 2024-05-15T15:26:59.247 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:26:58 smithi043 ceph-mon[24430]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:59.248 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:26:58 smithi043 ceph-mon[29127]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:26:59.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:26:58 smithi053 ceph-mon[25205]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:00.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:00 smithi043 ceph-mon[29127]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:00 smithi043 ceph-mon[24430]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:00 smithi053 ceph-mon[25205]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:02.210 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:02 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:02.210 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:02 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:02.210 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:02 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:02.210 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:02 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:02.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:02 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:02.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:02 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:02.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:02 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:02.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:02 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:02.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:02 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:02.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:02 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:02.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:02 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:02.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:02 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:03.234 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:03 smithi043 ceph-mon[24430]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:03.234 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:03 smithi043 ceph-mon[29127]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:03.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:03 smithi053 ceph-mon[25205]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:04.144 INFO:teuthology.orchestra.run.smithi043.stdout:Created osd(s) 1 on host 'smithi043' 2024-05-15T15:27:04.743 DEBUG:teuthology.orchestra.run.smithi043:osd.1> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.1.service 2024-05-15T15:27:04.746 INFO:tasks.cephadm:Deploying osd.2 on smithi043 with /dev/vg_nvme/lv_2... 2024-05-15T15:27:04.746 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-15T15:27:05.261 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:05 smithi043 ceph-mon[24430]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:05.261 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:05 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:05.261 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:05 smithi043 ceph-mon[24430]: from='osd.1 [v2:172.21.15.43:6810/3463535224,v1:172.21.15.43:6811/3463535224]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-15T15:27:05.261 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:05 smithi043 ceph-mon[24430]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-15T15:27:05.262 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:05 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:05.262 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:05 smithi043 ceph-mon[29127]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:05.262 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:05 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:05.262 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:05 smithi043 ceph-mon[29127]: from='osd.1 [v2:172.21.15.43:6810/3463535224,v1:172.21.15.43:6811/3463535224]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-15T15:27:05.262 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:05 smithi043 ceph-mon[29127]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-15T15:27:05.262 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:05 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:05.445 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:05 smithi053 ceph-mon[25205]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:05.445 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:05 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:05.445 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:05 smithi053 ceph-mon[25205]: from='osd.1 [v2:172.21.15.43:6810/3463535224,v1:172.21.15.43:6811/3463535224]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-15T15:27:05.445 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:05 smithi053 ceph-mon[25205]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-15T15:27:05.445 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:05 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:06.416 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:06 smithi043 ceph-mon[24430]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-15T15:27:06.416 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:06 smithi043 ceph-mon[24430]: osdmap e11: 2 total, 1 up, 2 in 2024-05-15T15:27:06.416 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:06 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:06.416 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:06 smithi043 ceph-mon[24430]: from='osd.1 [v2:172.21.15.43:6810/3463535224,v1:172.21.15.43:6811/3463535224]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:06.416 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:06 smithi043 ceph-mon[24430]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:06.416 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:06 smithi043 ceph-mon[29127]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-15T15:27:06.416 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:06 smithi043 ceph-mon[29127]: osdmap e11: 2 total, 1 up, 2 in 2024-05-15T15:27:06.416 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:06 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:06.417 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:06 smithi043 ceph-mon[29127]: from='osd.1 [v2:172.21.15.43:6810/3463535224,v1:172.21.15.43:6811/3463535224]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:06.417 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:06 smithi043 ceph-mon[29127]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:06.417 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:27:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[34927]: 2024-05-15T15:27:06.152+0000 7fea617a5700 -1 osd.1 0 waiting for initial osdmap 2024-05-15T15:27:06.417 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:27:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[34927]: 2024-05-15T15:27:06.155+0000 7fea5c13c700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:27:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:06 smithi053 ceph-mon[25205]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-15T15:27:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:06 smithi053 ceph-mon[25205]: osdmap e11: 2 total, 1 up, 2 in 2024-05-15T15:27:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:06 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:06 smithi053 ceph-mon[25205]: from='osd.1 [v2:172.21.15.43:6810/3463535224,v1:172.21.15.43:6811/3463535224]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:06 smithi053 ceph-mon[25205]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:07.167 INFO:teuthology.orchestra.run.smithi043.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-15T15:27:07.167 INFO:teuthology.orchestra.run.smithi043.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-15T15:27:07.167 INFO:teuthology.orchestra.run.smithi043.stderr: stderr: 10+0 records in 2024-05-15T15:27:07.167 INFO:teuthology.orchestra.run.smithi043.stderr:10+0 records out 2024-05-15T15:27:07.167 INFO:teuthology.orchestra.run.smithi043.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0113408 s, 925 MB/s 2024-05-15T15:27:07.167 INFO:teuthology.orchestra.run.smithi043.stderr:--> Zapping successful for: 2024-05-15T15:27:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:07 smithi043 ceph-mon[24430]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:07 smithi043 ceph-mon[24430]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:27:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:07 smithi043 ceph-mon[24430]: osdmap e12: 2 total, 1 up, 2 in 2024-05-15T15:27:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:07 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:07 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:07 smithi043 ceph-mon[29127]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:07 smithi043 ceph-mon[29127]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:27:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:07 smithi043 ceph-mon[29127]: osdmap e12: 2 total, 1 up, 2 in 2024-05-15T15:27:07.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:07 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:07.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:07 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:07 smithi053 ceph-mon[25205]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:07 smithi053 ceph-mon[25205]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:27:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:07 smithi053 ceph-mon[25205]: osdmap e12: 2 total, 1 up, 2 in 2024-05-15T15:27:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:07 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:07.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:07 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:07.913 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch daemon add osd smithi043:vg_nvme/lv_2 2024-05-15T15:27:08.523 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:08 smithi043 ceph-mon[24430]: purged_snaps scrub starts 2024-05-15T15:27:08.524 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:08 smithi043 ceph-mon[24430]: purged_snaps scrub ok 2024-05-15T15:27:08.524 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:08 smithi043 ceph-mon[24430]: osd.1 [v2:172.21.15.43:6810/3463535224,v1:172.21.15.43:6811/3463535224] boot 2024-05-15T15:27:08.524 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:08 smithi043 ceph-mon[24430]: osdmap e13: 2 total, 2 up, 2 in 2024-05-15T15:27:08.524 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:08.524 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:08 smithi043 ceph-mon[29127]: purged_snaps scrub starts 2024-05-15T15:27:08.524 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:08 smithi043 ceph-mon[29127]: purged_snaps scrub ok 2024-05-15T15:27:08.524 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:08 smithi043 ceph-mon[29127]: osd.1 [v2:172.21.15.43:6810/3463535224,v1:172.21.15.43:6811/3463535224] boot 2024-05-15T15:27:08.524 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:08 smithi043 ceph-mon[29127]: osdmap e13: 2 total, 2 up, 2 in 2024-05-15T15:27:08.524 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:08.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:08 smithi053 ceph-mon[25205]: purged_snaps scrub starts 2024-05-15T15:27:08.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:08 smithi053 ceph-mon[25205]: purged_snaps scrub ok 2024-05-15T15:27:08.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:08 smithi053 ceph-mon[25205]: osd.1 [v2:172.21.15.43:6810/3463535224,v1:172.21.15.43:6811/3463535224] boot 2024-05-15T15:27:08.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:08 smithi053 ceph-mon[25205]: osdmap e13: 2 total, 2 up, 2 in 2024-05-15T15:27:08.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:27:09.457 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:09 smithi043 ceph-mon[24430]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:09.457 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:09 smithi043 ceph-mon[24430]: osdmap e14: 2 total, 2 up, 2 in 2024-05-15T15:27:09.457 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:09 smithi043 ceph-mon[29127]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:09.458 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:09 smithi043 ceph-mon[29127]: osdmap e14: 2 total, 2 up, 2 in 2024-05-15T15:27:09.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:09 smithi053 ceph-mon[25205]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-15T15:27:09.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:09 smithi053 ceph-mon[25205]: osdmap e14: 2 total, 2 up, 2 in 2024-05-15T15:27:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:10 smithi043 ceph-mon[24430]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:10 smithi043 ceph-mon[24430]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:27:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:27:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:27:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:10 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:10.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:10 smithi043 ceph-mon[29127]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:10.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:10 smithi043 ceph-mon[29127]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:27:10.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:27:10.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:27:10.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:10 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:10 smithi053 ceph-mon[25205]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:10 smithi053 ceph-mon[25205]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:27:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:27:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:27:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:10 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:12.155 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: Detected new or changed devices on smithi043 2024-05-15T15:27:12.156 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.156 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:12.156 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:12.156 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: Adjusting osd_memory_target on smithi043 to 8034M 2024-05-15T15:27:12.156 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.156 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.156 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:12.156 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:12.156 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:12.156 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: Detected new or changed devices on smithi043 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: Adjusting osd_memory_target on smithi043 to 8034M 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:12.157 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: Detected new or changed devices on smithi043 2024-05-15T15:27:12.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:12.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:12.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: Adjusting osd_memory_target on smithi043 to 8034M 2024-05-15T15:27:12.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:12.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:12 smithi043 ceph-mon[24430]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:12 smithi043 ceph-mon[29127]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:13.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:12 smithi053 ceph-mon[25205]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/271062201' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243"}]: dispatch 2024-05-15T15:27:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243"}]: dispatch 2024-05-15T15:27:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243"}]': finished 2024-05-15T15:27:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[24430]: osdmap e15: 3 total, 2 up, 3 in 2024-05-15T15:27:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1410002976' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:27:14.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/271062201' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243"}]: dispatch 2024-05-15T15:27:14.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243"}]: dispatch 2024-05-15T15:27:14.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243"}]': finished 2024-05-15T15:27:14.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[29127]: osdmap e15: 3 total, 2 up, 3 in 2024-05-15T15:27:14.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:14.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:13 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1410002976' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:27:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:13 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/271062201' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243"}]: dispatch 2024-05-15T15:27:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:13 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243"}]: dispatch 2024-05-15T15:27:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:13 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243"}]': finished 2024-05-15T15:27:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:13 smithi053 ceph-mon[25205]: osdmap e15: 3 total, 2 up, 3 in 2024-05-15T15:27:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:13 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:13 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1410002976' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:27:15.490 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:15 smithi043 ceph-mon[24430]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:15.491 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:15 smithi043 ceph-mon[29127]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:15.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:15 smithi053 ceph-mon[25205]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:17.333 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:17 smithi043 ceph-mon[24430]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:17.334 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:17 smithi043 ceph-mon[29127]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:17 smithi053 ceph-mon[25205]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:19.484 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:19 smithi043 ceph-mon[24430]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:19.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:19 smithi043 ceph-mon[29127]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:19 smithi053 ceph-mon[25205]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:20 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-15T15:27:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:20 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:20 smithi043 ceph-mon[24430]: Deploying daemon osd.2 on smithi043 2024-05-15T15:27:20.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:20 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-15T15:27:20.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:20 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:20.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:20 smithi043 ceph-mon[29127]: Deploying daemon osd.2 on smithi043 2024-05-15T15:27:20.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:20 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-15T15:27:20.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:20 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:20.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:20 smithi053 ceph-mon[25205]: Deploying daemon osd.2 on smithi043 2024-05-15T15:27:21.558 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:21 smithi043 ceph-mon[24430]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:21.558 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:21 smithi043 ceph-mon[29127]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:21 smithi053 ceph-mon[25205]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:23.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:23 smithi043 ceph-mon[24430]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:23.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:23 smithi043 ceph-mon[29127]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:23.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:23 smithi053 ceph-mon[25205]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:24.503 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:27:24.503 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:27:24.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:24 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:27:24.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:24 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:27:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:24 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:27:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:24 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:27:25.479 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:25 smithi043 ceph-mon[24430]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:25.479 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:25.479 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:25.479 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:25.479 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:25.480 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:25 smithi043 ceph-mon[29127]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:25.480 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:25 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:25.480 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:25 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:25.480 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:25 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:25.480 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:25 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:25 smithi053 ceph-mon[25205]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:25 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:25 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:25 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:25 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:27.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:27 smithi043 ceph-mon[24430]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:27.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:27 smithi043 ceph-mon[29127]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:27 smithi053 ceph-mon[25205]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:27.672 INFO:teuthology.orchestra.run.smithi043.stdout:Created osd(s) 2 on host 'smithi043' 2024-05-15T15:27:28.287 DEBUG:teuthology.orchestra.run.smithi043:osd.2> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.2.service 2024-05-15T15:27:28.290 INFO:tasks.cephadm:Deploying osd.3 on smithi043 with /dev/vg_nvme/lv_1... 2024-05-15T15:27:28.290 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-15T15:27:28.907 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:28 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:28.907 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:28 smithi043 ceph-mon[24430]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:28.907 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:28 smithi043 ceph-mon[24430]: from='osd.2 [v2:172.21.15.43:6818/50306098,v1:172.21.15.43:6819/50306098]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:27:28.907 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:28 smithi043 ceph-mon[24430]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:27:28.907 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:28 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:28.907 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:28 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:28.907 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:28 smithi043 ceph-mon[29127]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:28.907 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:28 smithi043 ceph-mon[29127]: from='osd.2 [v2:172.21.15.43:6818/50306098,v1:172.21.15.43:6819/50306098]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:27:28.907 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:28 smithi043 ceph-mon[29127]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:27:28.908 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:28 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:29.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:28 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:29.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:28 smithi053 ceph-mon[25205]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:29.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:28 smithi053 ceph-mon[25205]: from='osd.2 [v2:172.21.15.43:6818/50306098,v1:172.21.15.43:6819/50306098]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:27:29.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:28 smithi053 ceph-mon[25205]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:27:29.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:28 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:30.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:29 smithi043 ceph-mon[29127]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-15T15:27:30.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:29 smithi043 ceph-mon[29127]: osdmap e16: 3 total, 2 up, 3 in 2024-05-15T15:27:30.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:29 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:30.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:29 smithi043 ceph-mon[29127]: from='osd.2 [v2:172.21.15.43:6818/50306098,v1:172.21.15.43:6819/50306098]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:30.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:29 smithi043 ceph-mon[29127]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:29 smithi043 ceph-mon[24430]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-15T15:27:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:29 smithi043 ceph-mon[24430]: osdmap e16: 3 total, 2 up, 3 in 2024-05-15T15:27:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:29 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:29 smithi043 ceph-mon[24430]: from='osd.2 [v2:172.21.15.43:6818/50306098,v1:172.21.15.43:6819/50306098]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:29 smithi043 ceph-mon[24430]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:30.087 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:27:29 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[37838]: 2024-05-15T15:27:29.679+0000 7ff807acb700 -1 osd.2 0 waiting for initial osdmap 2024-05-15T15:27:30.087 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:27:29 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[37838]: 2024-05-15T15:27:29.682+0000 7ff802c63700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:27:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:29 smithi053 ceph-mon[25205]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-15T15:27:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:29 smithi053 ceph-mon[25205]: osdmap e16: 3 total, 2 up, 3 in 2024-05-15T15:27:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:29 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:29 smithi053 ceph-mon[25205]: from='osd.2 [v2:172.21.15.43:6818/50306098,v1:172.21.15.43:6819/50306098]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:29 smithi053 ceph-mon[25205]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:30.967 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:30 smithi043 ceph-mon[24430]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:27:30.967 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:30 smithi043 ceph-mon[24430]: osdmap e17: 3 total, 2 up, 3 in 2024-05-15T15:27:30.967 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:30 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:30.967 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:30 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:30.968 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:30 smithi043 ceph-mon[24430]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:30.968 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:30 smithi043 ceph-mon[29127]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:27:30.968 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:30 smithi043 ceph-mon[29127]: osdmap e17: 3 total, 2 up, 3 in 2024-05-15T15:27:30.968 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:30 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:30.968 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:30 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:30.968 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:30 smithi043 ceph-mon[29127]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:30 smithi053 ceph-mon[25205]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:27:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:30 smithi053 ceph-mon[25205]: osdmap e17: 3 total, 2 up, 3 in 2024-05-15T15:27:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:30 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:30 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:30 smithi053 ceph-mon[25205]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-15T15:27:31.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:31 smithi043 ceph-mon[29127]: purged_snaps scrub starts 2024-05-15T15:27:31.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:31 smithi043 ceph-mon[29127]: purged_snaps scrub ok 2024-05-15T15:27:31.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:31 smithi043 ceph-mon[29127]: osd.2 [v2:172.21.15.43:6818/50306098,v1:172.21.15.43:6819/50306098] boot 2024-05-15T15:27:31.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:31 smithi043 ceph-mon[29127]: osdmap e18: 3 total, 3 up, 3 in 2024-05-15T15:27:31.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:31 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:31 smithi043 ceph-mon[24430]: purged_snaps scrub starts 2024-05-15T15:27:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:31 smithi043 ceph-mon[24430]: purged_snaps scrub ok 2024-05-15T15:27:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:31 smithi043 ceph-mon[24430]: osd.2 [v2:172.21.15.43:6818/50306098,v1:172.21.15.43:6819/50306098] boot 2024-05-15T15:27:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:31 smithi043 ceph-mon[24430]: osdmap e18: 3 total, 3 up, 3 in 2024-05-15T15:27:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:31 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:31.920 INFO:teuthology.orchestra.run.smithi043.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-15T15:27:31.921 INFO:teuthology.orchestra.run.smithi043.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-15T15:27:31.921 INFO:teuthology.orchestra.run.smithi043.stderr: stderr: 10+0 records in 2024-05-15T15:27:31.921 INFO:teuthology.orchestra.run.smithi043.stderr:10+0 records out 2024-05-15T15:27:31.921 INFO:teuthology.orchestra.run.smithi043.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0100475 s, 1.0 GB/s 2024-05-15T15:27:31.921 INFO:teuthology.orchestra.run.smithi043.stderr:--> Zapping successful for: 2024-05-15T15:27:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:31 smithi053 ceph-mon[25205]: purged_snaps scrub starts 2024-05-15T15:27:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:31 smithi053 ceph-mon[25205]: purged_snaps scrub ok 2024-05-15T15:27:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:31 smithi053 ceph-mon[25205]: osd.2 [v2:172.21.15.43:6818/50306098,v1:172.21.15.43:6819/50306098] boot 2024-05-15T15:27:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:31 smithi053 ceph-mon[25205]: osdmap e18: 3 total, 3 up, 3 in 2024-05-15T15:27:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:31 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:27:32.634 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch daemon add osd smithi043:vg_nvme/lv_1 2024-05-15T15:27:33.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:32 smithi043 ceph-mon[24430]: osdmap e19: 3 total, 3 up, 3 in 2024-05-15T15:27:33.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:32 smithi043 ceph-mon[24430]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:33.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:32 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-15T15:27:33.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:32 smithi043 ceph-mon[29127]: osdmap e19: 3 total, 3 up, 3 in 2024-05-15T15:27:33.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:32 smithi043 ceph-mon[29127]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:33.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:32 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-15T15:27:33.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:32 smithi053 ceph-mon[25205]: osdmap e19: 3 total, 3 up, 3 in 2024-05-15T15:27:33.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:32 smithi053 ceph-mon[25205]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:33.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:32 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-15T15:27:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-15T15:27:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: osdmap e20: 3 total, 3 up, 3 in 2024-05-15T15:27:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-15T15:27:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: Detected new or changed devices on smithi043 2024-05-15T15:27:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: Adjusting osd_memory_target on smithi043 to 5356M 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: osdmap e20: 3 total, 3 up, 3 in 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: Detected new or changed devices on smithi043 2024-05-15T15:27:34.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:34.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:34.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:34.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: Adjusting osd_memory_target on smithi043 to 5356M 2024-05-15T15:27:34.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:34.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:34.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:34.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:33 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-15T15:27:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: osdmap e20: 3 total, 3 up, 3 in 2024-05-15T15:27:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-15T15:27:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: Detected new or changed devices on smithi043 2024-05-15T15:27:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:34.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:34.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:34.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: Adjusting osd_memory_target on smithi043 to 5356M 2024-05-15T15:27:34.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:34.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:34.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:34.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:33 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:34.706 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:27:34 smithi043 sudo[39146]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-15T15:27:34.706 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:27:34 smithi043 sudo[39146]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:34.706 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:27:34 smithi043 sudo[39146]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:35.031 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:34 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-15T15:27:35.031 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:34 smithi053 ceph-mon[25205]: osdmap e21: 3 total, 3 up, 3 in 2024-05-15T15:27:35.031 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:34 smithi053 ceph-mon[25205]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:35.085 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:34 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-15T15:27:35.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:34 smithi043 ceph-mon[29127]: osdmap e21: 3 total, 3 up, 3 in 2024-05-15T15:27:35.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:34 smithi043 ceph-mon[29127]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:35.086 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:27:34 smithi043 sudo[39208]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-15T15:27:35.086 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:27:34 smithi043 sudo[39208]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:35.086 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:27:34 smithi043 sudo[39208]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:35.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:34 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-15T15:27:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:34 smithi043 ceph-mon[24430]: osdmap e21: 3 total, 3 up, 3 in 2024-05-15T15:27:35.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:34 smithi043 ceph-mon[24430]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:35.466 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:27:35 smithi043 sudo[39254]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-15T15:27:35.466 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:27:35 smithi043 sudo[39254]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:35.466 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:27:35 smithi043 sudo[39254]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:35.720 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[29127]: osdmap e22: 3 total, 3 up, 3 in 2024-05-15T15:27:35.720 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[29127]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:27:35.720 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:27:35.721 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:27:35.721 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:35.721 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:35.721 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[24430]: osdmap e22: 3 total, 3 up, 3 in 2024-05-15T15:27:35.721 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[24430]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:27:35.721 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:27:35.721 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:27:35.721 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:35.721 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:35 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:36.036 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:27:35 smithi043 sudo[39274]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-15T15:27:36.036 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:27:35 smithi043 sudo[39274]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:36.036 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:27:35 smithi043 sudo[39274]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:36.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:35 smithi053 ceph-mon[25205]: osdmap e22: 3 total, 3 up, 3 in 2024-05-15T15:27:36.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:35 smithi053 ceph-mon[25205]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi043:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:27:36.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:35 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:27:36.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:35 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:27:36.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:35 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:36.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:35 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:36.336 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:27:36 smithi043 sudo[39296]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-15T15:27:36.336 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:27:36 smithi043 sudo[39296]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:36.336 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:27:36 smithi043 sudo[39296]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:37.086 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:27:36 smithi043 sudo[39315]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-15T15:27:37.086 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:27:36 smithi043 sudo[39315]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:37.086 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:27:36 smithi043 sudo[39315]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:37.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:37 smithi043 sudo[39343]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-15T15:27:37.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:37 smithi043 sudo[39343]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:37.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:37 smithi043 sudo[39343]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:37 smithi053 ceph-mon[25205]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:37 smithi053 ceph-mon[25205]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-15T15:27:38.047 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:37 smithi043 ceph-mon[29127]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:38.047 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:37 smithi043 ceph-mon[29127]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-15T15:27:38.047 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:37 smithi043 sudo[39346]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-15T15:27:38.047 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:37 smithi043 sudo[39346]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:38.047 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:37 smithi043 sudo[39346]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:38.047 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:37 smithi043 ceph-mon[24430]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:38.047 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:37 smithi043 ceph-mon[24430]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-15T15:27:38.335 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:38 smithi043 sudo[39349]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-15T15:27:38.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:38 smithi043 sudo[39349]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:38.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:38 smithi043 sudo[39349]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:38.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:38 smithi043 sudo[39352]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-15T15:27:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:38 smithi043 sudo[39352]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:38 smithi043 sudo[39352]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:39.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:38 smithi053 sudo[26648]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-15T15:27:39.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:38 smithi053 sudo[26648]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:39.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:38 smithi053 sudo[26648]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/4259066205' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f79328f3-261d-4a48-8a97-952a3bf9b97e"}]: dispatch 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f79328f3-261d-4a48-8a97-952a3bf9b97e"}]: dispatch 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f79328f3-261d-4a48-8a97-952a3bf9b97e"}]': finished 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: osdmap e23: 4 total, 3 up, 4 in 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:27:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/4259066205' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f79328f3-261d-4a48-8a97-952a3bf9b97e"}]: dispatch 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f79328f3-261d-4a48-8a97-952a3bf9b97e"}]: dispatch 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:27:39.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f79328f3-261d-4a48-8a97-952a3bf9b97e"}]': finished 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: osdmap e23: 4 total, 3 up, 4 in 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:27:39.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:27:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 sudo[26651]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-15T15:27:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 sudo[26651]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-15T15:27:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 sudo[26651]: pam_unix(sudo:session): session closed for user root 2024-05-15T15:27:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/4259066205' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f79328f3-261d-4a48-8a97-952a3bf9b97e"}]: dispatch 2024-05-15T15:27:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-15T15:27:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:27:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f79328f3-261d-4a48-8a97-952a3bf9b97e"}]: dispatch 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: mgrmap e15: y(active, since 2m), standbys: x 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f79328f3-261d-4a48-8a97-952a3bf9b97e"}]': finished 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: osdmap e23: 4 total, 3 up, 4 in 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:27:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:27:39.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:27:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:40 smithi043 ceph-mon[29127]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-15T15:27:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:40 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1944803449' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:27:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:40 smithi043 ceph-mon[29127]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-15T15:27:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:40 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:40 smithi043 ceph-mon[24430]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-15T15:27:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:40 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1944803449' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:27:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:40 smithi043 ceph-mon[24430]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-15T15:27:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:40 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:40 smithi053 ceph-mon[25205]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-15T15:27:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:40 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1944803449' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:27:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:40 smithi053 ceph-mon[25205]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-15T15:27:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:40 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:41.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:41 smithi043 ceph-mon[24430]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:41.674 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:41 smithi043 ceph-mon[29127]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:41 smithi053 ceph-mon[25205]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:43.744 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:43 smithi043 ceph-mon[24430]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:43.744 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:43 smithi043 ceph-mon[29127]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:43 smithi053 ceph-mon[25205]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:45.624 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:45 smithi043 ceph-mon[24430]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-05-15T15:27:45.624 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:45 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-15T15:27:45.624 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:45 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:45.625 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:45 smithi043 ceph-mon[29127]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-05-15T15:27:45.625 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:45 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-15T15:27:45.625 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:45 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:45 smithi053 ceph-mon[25205]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-05-15T15:27:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:45 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-15T15:27:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:45 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:46.739 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:46 smithi043 ceph-mon[24430]: Deploying daemon osd.3 on smithi043 2024-05-15T15:27:46.739 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:46 smithi043 ceph-mon[29127]: Deploying daemon osd.3 on smithi043 2024-05-15T15:27:46.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:46 smithi053 ceph-mon[25205]: Deploying daemon osd.3 on smithi043 2024-05-15T15:27:47.823 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:47 smithi043 ceph-mon[24430]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-05-15T15:27:47.824 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:47 smithi043 ceph-mon[29127]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-05-15T15:27:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:47 smithi053 ceph-mon[25205]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-05-15T15:27:49.715 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:49 smithi043 ceph-mon[24430]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-05-15T15:27:49.716 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:49 smithi043 ceph-mon[29127]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-05-15T15:27:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:49 smithi053 ceph-mon[25205]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-05-15T15:27:51.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:51 smithi053 ceph-mon[25205]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-05-15T15:27:51.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:51 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:51.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:51 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:51.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:51 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:51.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:51 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:51.382 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:51 smithi043 ceph-mon[24430]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-05-15T15:27:51.383 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:51.383 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:51.383 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:51.383 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:51 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:51.383 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:51 smithi043 ceph-mon[29127]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-05-15T15:27:51.383 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:51 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:51.383 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:51 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:51.383 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:51 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:51.383 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:51 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:53.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:53 smithi053 ceph-mon[25205]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-05-15T15:27:53.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:53 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:53.458 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:53 smithi043 ceph-mon[24430]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-05-15T15:27:53.458 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:53 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:53.458 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:53 smithi043 ceph-mon[29127]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-05-15T15:27:53.458 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:53 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:53.487 INFO:teuthology.orchestra.run.smithi043.stdout:Created osd(s) 3 on host 'smithi043' 2024-05-15T15:27:54.136 DEBUG:teuthology.orchestra.run.smithi043:osd.3> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.3.service 2024-05-15T15:27:54.285 INFO:tasks.cephadm:Deploying osd.4 on smithi053 with /dev/vg_nvme/lv_4... 2024-05-15T15:27:54.285 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-15T15:27:54.371 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[24430]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-15T15:27:54.371 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:54.372 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:54.372 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:54.372 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:54.372 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:54.372 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[29127]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-15T15:27:54.372 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:54.372 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:54.372 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:54.372 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:54.373 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:54 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:54.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:54 smithi053 ceph-mon[25205]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-15T15:27:54.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:54 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:54.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:54 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:54.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:54 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:27:54.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:54 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:54.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:54 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:27:55.151 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:55 smithi053 ceph-mon[25205]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-05-15T15:27:55.151 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:55 smithi053 ceph-mon[25205]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-15T15:27:55.152 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:55 smithi053 ceph-mon[25205]: osdmap e24: 4 total, 3 up, 4 in 2024-05-15T15:27:55.152 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:55 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:55.152 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:55 smithi053 ceph-mon[25205]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:55.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:55 smithi043 ceph-mon[24430]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-05-15T15:27:55.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:55 smithi043 ceph-mon[24430]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-15T15:27:55.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:55 smithi043 ceph-mon[24430]: osdmap e24: 4 total, 3 up, 4 in 2024-05-15T15:27:55.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:55 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:55.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:55 smithi043 ceph-mon[24430]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:55.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:55 smithi043 ceph-mon[29127]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-05-15T15:27:55.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:55 smithi043 ceph-mon[29127]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-15T15:27:55.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:55 smithi043 ceph-mon[29127]: osdmap e24: 4 total, 3 up, 4 in 2024-05-15T15:27:55.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:55 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:55.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:55 smithi043 ceph-mon[29127]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:27:55.337 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:27:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[40799]: 2024-05-15T15:27:55.102+0000 7f0cb55c1700 -1 osd.3 0 waiting for initial osdmap 2024-05-15T15:27:55.337 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:27:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[40799]: 2024-05-15T15:27:55.106+0000 7f0caff58700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:27:56.193 INFO:teuthology.orchestra.run.smithi053.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-15T15:27:56.193 INFO:teuthology.orchestra.run.smithi053.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-15T15:27:56.194 INFO:teuthology.orchestra.run.smithi053.stderr: stderr: 10+0 records in 2024-05-15T15:27:56.194 INFO:teuthology.orchestra.run.smithi053.stderr:10+0 records out 2024-05-15T15:27:56.194 INFO:teuthology.orchestra.run.smithi053.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00836126 s, 1.3 GB/s 2024-05-15T15:27:56.194 INFO:teuthology.orchestra.run.smithi053.stderr:--> Zapping successful for: 2024-05-15T15:27:56.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:56 smithi053 ceph-mon[25205]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:27:56.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:56 smithi053 ceph-mon[25205]: osdmap e25: 4 total, 3 up, 4 in 2024-05-15T15:27:56.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:56 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:56.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:56 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:56.395 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:56 smithi043 ceph-mon[24430]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:27:56.395 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:56 smithi043 ceph-mon[24430]: osdmap e25: 4 total, 3 up, 4 in 2024-05-15T15:27:56.395 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:56 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:56.395 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:56 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:56.395 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:56 smithi043 ceph-mon[29127]: from='osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]': finished 2024-05-15T15:27:56.395 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:56 smithi043 ceph-mon[29127]: osdmap e25: 4 total, 3 up, 4 in 2024-05-15T15:27:56.396 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:56 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:56.396 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:56 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:56.793 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch daemon add osd smithi053:vg_nvme/lv_4 2024-05-15T15:27:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: purged_snaps scrub starts 2024-05-15T15:27:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: purged_snaps scrub ok 2024-05-15T15:27:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486] boot 2024-05-15T15:27:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: osdmap e26: 4 total, 4 up, 4 in 2024-05-15T15:27:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: purged_snaps scrub starts 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: purged_snaps scrub ok 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:57.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486] boot 2024-05-15T15:27:57.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: osdmap e26: 4 total, 4 up, 4 in 2024-05-15T15:27:57.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:57.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:57.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:57.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:57 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: purged_snaps scrub starts 2024-05-15T15:27:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: purged_snaps scrub ok 2024-05-15T15:27:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-15T15:27:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: osd.3 [v2:172.21.15.43:6826/3944497486,v1:172.21.15.43:6827/3944497486] boot 2024-05-15T15:27:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: osdmap e26: 4 total, 4 up, 4 in 2024-05-15T15:27:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:27:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:27:57.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:57.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:57 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:27:58.354 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:58 smithi053 ceph-mon[25205]: Detected new or changed devices on smithi043 2024-05-15T15:27:58.354 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:58 smithi053 ceph-mon[25205]: Adjusting osd_memory_target on smithi043 to 4017M 2024-05-15T15:27:58.354 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:58 smithi053 ceph-mon[25205]: osdmap e27: 4 total, 4 up, 4 in 2024-05-15T15:27:58.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:58 smithi043 ceph-mon[29127]: Detected new or changed devices on smithi043 2024-05-15T15:27:58.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:58 smithi043 ceph-mon[29127]: Adjusting osd_memory_target on smithi043 to 4017M 2024-05-15T15:27:58.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:58 smithi043 ceph-mon[29127]: osdmap e27: 4 total, 4 up, 4 in 2024-05-15T15:27:58.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:58 smithi043 ceph-mon[24430]: Detected new or changed devices on smithi043 2024-05-15T15:27:58.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:58 smithi043 ceph-mon[24430]: Adjusting osd_memory_target on smithi043 to 4017M 2024-05-15T15:27:58.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:58 smithi043 ceph-mon[24430]: osdmap e27: 4 total, 4 up, 4 in 2024-05-15T15:27:59.316 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:59 smithi053 ceph-mon[25205]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:27:59.316 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:59 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:27:59.317 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:59 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:27:59.317 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:27:59 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:59 smithi043 ceph-mon[24430]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:27:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:27:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:27:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:27:59 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:27:59.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:59 smithi043 ceph-mon[29127]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:27:59.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:27:59.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:27:59.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:27:59 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:00.305 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:00 smithi053 ceph-mon[25205]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:28:00.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:00 smithi043 ceph-mon[24430]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:28:00.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:00 smithi043 ceph-mon[29127]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:28:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:01 smithi053 ceph-mon[25205]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:01 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/1647891244' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50ed214e-19e1-4bab-95ac-d636276f8925"}]: dispatch 2024-05-15T15:28:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:01 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50ed214e-19e1-4bab-95ac-d636276f8925"}]: dispatch 2024-05-15T15:28:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:01 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50ed214e-19e1-4bab-95ac-d636276f8925"}]': finished 2024-05-15T15:28:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:01 smithi053 ceph-mon[25205]: osdmap e28: 5 total, 4 up, 5 in 2024-05-15T15:28:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:01 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[24430]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/1647891244' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50ed214e-19e1-4bab-95ac-d636276f8925"}]: dispatch 2024-05-15T15:28:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50ed214e-19e1-4bab-95ac-d636276f8925"}]: dispatch 2024-05-15T15:28:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50ed214e-19e1-4bab-95ac-d636276f8925"}]': finished 2024-05-15T15:28:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[24430]: osdmap e28: 5 total, 4 up, 5 in 2024-05-15T15:28:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:01.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[29127]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/1647891244' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50ed214e-19e1-4bab-95ac-d636276f8925"}]: dispatch 2024-05-15T15:28:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50ed214e-19e1-4bab-95ac-d636276f8925"}]: dispatch 2024-05-15T15:28:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50ed214e-19e1-4bab-95ac-d636276f8925"}]': finished 2024-05-15T15:28:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[29127]: osdmap e28: 5 total, 4 up, 5 in 2024-05-15T15:28:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:01 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:02.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:02 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/3657478516' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:28:02.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:02 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/3657478516' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:28:02.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:02 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/3657478516' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:28:03.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:03 smithi053 ceph-mon[25205]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:03 smithi043 ceph-mon[24430]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:03.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:03 smithi043 ceph-mon[29127]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:05.351 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:05 smithi053 ceph-mon[25205]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:05.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:05 smithi043 ceph-mon[24430]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:05.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:05 smithi043 ceph-mon[29127]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:07.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:07 smithi053 ceph-mon[25205]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:07.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:07 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-15T15:28:07.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:07 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:07.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:07 smithi043 ceph-mon[24430]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:07.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:07 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-15T15:28:07.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:07 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:07.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:07 smithi043 ceph-mon[29127]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:07.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:07 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-15T15:28:07.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:07 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:08.346 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:08 smithi053 ceph-mon[25205]: Deploying daemon osd.4 on smithi053 2024-05-15T15:28:08.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:08 smithi043 ceph-mon[24430]: Deploying daemon osd.4 on smithi053 2024-05-15T15:28:08.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:08 smithi043 ceph-mon[29127]: Deploying daemon osd.4 on smithi053 2024-05-15T15:28:09.548 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:09 smithi053 ceph-mon[25205]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:09.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:09 smithi043 ceph-mon[24430]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:09.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:09 smithi043 ceph-mon[29127]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:11.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:10 smithi043 ceph-mon[24430]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:11.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:10 smithi043 ceph-mon[29127]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:11.093 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:10 smithi053 ceph-mon[25205]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:13.312 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:12 smithi053 ceph-mon[25205]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:13.312 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:12 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:13.312 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:12 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:13.313 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:12 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:13.313 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:12 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:13.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:12 smithi043 ceph-mon[24430]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:12 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:12 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:12 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:12 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:12 smithi043 ceph-mon[29127]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:12 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:12 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:12 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:12 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:15.109 INFO:teuthology.orchestra.run.smithi053.stdout:Created osd(s) 4 on host 'smithi053' 2024-05-15T15:28:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:15 smithi053 ceph-mon[25205]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:15 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:15 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:15 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:15 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:15 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:15 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:15 smithi053 ceph-mon[25205]: from='osd.4 [v2:172.21.15.53:6800/315819048,v1:172.21.15.53:6801/315819048]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T15:28:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:15 smithi053 ceph-mon[25205]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T15:28:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[24430]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[24430]: from='osd.4 [v2:172.21.15.53:6800/315819048,v1:172.21.15.53:6801/315819048]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T15:28:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[24430]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T15:28:15.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[29127]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:15.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:15.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:15.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:15.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:15.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:15.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:15.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[29127]: from='osd.4 [v2:172.21.15.53:6800/315819048,v1:172.21.15.53:6801/315819048]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T15:28:15.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:15 smithi043 ceph-mon[29127]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T15:28:15.691 DEBUG:teuthology.orchestra.run.smithi053:osd.4> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.4.service 2024-05-15T15:28:15.693 INFO:tasks.cephadm:Deploying osd.5 on smithi053 with /dev/vg_nvme/lv_3... 2024-05-15T15:28:15.693 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-15T15:28:17.376 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:17 smithi053 ceph-mon[25205]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:17.376 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:17 smithi053 ceph-mon[25205]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-15T15:28:17.376 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:17 smithi053 ceph-mon[25205]: osdmap e29: 5 total, 4 up, 5 in 2024-05-15T15:28:17.376 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:17 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:17.376 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:17 smithi053 ceph-mon[25205]: from='osd.4 [v2:172.21.15.53:6800/315819048,v1:172.21.15.53:6801/315819048]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:17.377 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:17 smithi053 ceph-mon[25205]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:17.377 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 15:28:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[28554]: 2024-05-15T15:28:17.115+0000 7f949bbed700 -1 osd.4 0 waiting for initial osdmap 2024-05-15T15:28:17.377 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 15:28:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[28554]: 2024-05-15T15:28:17.120+0000 7f9496584700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:28:17.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[24430]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:17.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[24430]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-15T15:28:17.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[24430]: osdmap e29: 5 total, 4 up, 5 in 2024-05-15T15:28:17.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:17.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[24430]: from='osd.4 [v2:172.21.15.53:6800/315819048,v1:172.21.15.53:6801/315819048]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:17.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[24430]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:17.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[29127]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:17.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[29127]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-15T15:28:17.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[29127]: osdmap e29: 5 total, 4 up, 5 in 2024-05-15T15:28:17.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:17.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[29127]: from='osd.4 [v2:172.21.15.53:6800/315819048,v1:172.21.15.53:6801/315819048]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:17.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:17 smithi043 ceph-mon[29127]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:18.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:18 smithi053 ceph-mon[25205]: purged_snaps scrub starts 2024-05-15T15:28:18.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:18 smithi053 ceph-mon[25205]: purged_snaps scrub ok 2024-05-15T15:28:18.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:18 smithi053 ceph-mon[25205]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:28:18.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:18 smithi053 ceph-mon[25205]: osdmap e30: 5 total, 4 up, 5 in 2024-05-15T15:28:18.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:18 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:18.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:18 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:18.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[24430]: purged_snaps scrub starts 2024-05-15T15:28:18.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[24430]: purged_snaps scrub ok 2024-05-15T15:28:18.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[24430]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:28:18.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[24430]: osdmap e30: 5 total, 4 up, 5 in 2024-05-15T15:28:18.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:18.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:18.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[29127]: purged_snaps scrub starts 2024-05-15T15:28:18.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[29127]: purged_snaps scrub ok 2024-05-15T15:28:18.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[29127]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:28:18.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[29127]: osdmap e30: 5 total, 4 up, 5 in 2024-05-15T15:28:18.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:18.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:18 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:19 smithi053 ceph-mon[25205]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:19 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:19 smithi053 ceph-mon[25205]: osd.4 [v2:172.21.15.53:6800/315819048,v1:172.21.15.53:6801/315819048] boot 2024-05-15T15:28:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:19 smithi053 ceph-mon[25205]: osdmap e31: 5 total, 5 up, 5 in 2024-05-15T15:28:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:19 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:19 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:19 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:28:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:19 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:19.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:19 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[24430]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[24430]: osd.4 [v2:172.21.15.53:6800/315819048,v1:172.21.15.53:6801/315819048] boot 2024-05-15T15:28:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[24430]: osdmap e31: 5 total, 5 up, 5 in 2024-05-15T15:28:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:28:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:19.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[29127]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-15T15:28:19.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:19.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[29127]: osd.4 [v2:172.21.15.53:6800/315819048,v1:172.21.15.53:6801/315819048] boot 2024-05-15T15:28:19.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[29127]: osdmap e31: 5 total, 5 up, 5 in 2024-05-15T15:28:19.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:28:19.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:19.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:28:19.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:19.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:19 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:19.677 INFO:teuthology.orchestra.run.smithi053.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-15T15:28:19.677 INFO:teuthology.orchestra.run.smithi053.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-15T15:28:19.677 INFO:teuthology.orchestra.run.smithi053.stderr: stderr: 10+0 records in 2024-05-15T15:28:19.677 INFO:teuthology.orchestra.run.smithi053.stderr:10+0 records out 2024-05-15T15:28:19.677 INFO:teuthology.orchestra.run.smithi053.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0093169 s, 1.1 GB/s 2024-05-15T15:28:19.677 INFO:teuthology.orchestra.run.smithi053.stderr:--> Zapping successful for: 2024-05-15T15:28:20.223 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:20 smithi053 ceph-mon[25205]: Detected new or changed devices on smithi053 2024-05-15T15:28:20.223 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:20 smithi053 ceph-mon[25205]: Adjusting osd_memory_target on smithi053 to 17093M 2024-05-15T15:28:20.223 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:20 smithi053 ceph-mon[25205]: osdmap e32: 5 total, 5 up, 5 in 2024-05-15T15:28:20.250 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch daemon add osd smithi053:vg_nvme/lv_3 2024-05-15T15:28:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:20 smithi043 ceph-mon[24430]: Detected new or changed devices on smithi053 2024-05-15T15:28:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:20 smithi043 ceph-mon[24430]: Adjusting osd_memory_target on smithi053 to 17093M 2024-05-15T15:28:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:20 smithi043 ceph-mon[24430]: osdmap e32: 5 total, 5 up, 5 in 2024-05-15T15:28:20.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:20 smithi043 ceph-mon[29127]: Detected new or changed devices on smithi053 2024-05-15T15:28:20.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:20 smithi043 ceph-mon[29127]: Adjusting osd_memory_target on smithi053 to 17093M 2024-05-15T15:28:20.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:20 smithi043 ceph-mon[29127]: osdmap e32: 5 total, 5 up, 5 in 2024-05-15T15:28:21.504 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:21 smithi053 ceph-mon[25205]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:21.504 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:21 smithi053 ceph-mon[25205]: osdmap e33: 5 total, 5 up, 5 in 2024-05-15T15:28:21.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:21 smithi043 ceph-mon[24430]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:21.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:21 smithi043 ceph-mon[24430]: osdmap e33: 5 total, 5 up, 5 in 2024-05-15T15:28:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:21 smithi043 ceph-mon[29127]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:21 smithi043 ceph-mon[29127]: osdmap e33: 5 total, 5 up, 5 in 2024-05-15T15:28:22.482 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:22 smithi053 ceph-mon[25205]: osdmap e34: 5 total, 5 up, 5 in 2024-05-15T15:28:22.482 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:22 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:28:22.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:22 smithi043 ceph-mon[24430]: osdmap e34: 5 total, 5 up, 5 in 2024-05-15T15:28:22.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:22 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:28:22.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:22 smithi043 ceph-mon[29127]: osdmap e34: 5 total, 5 up, 5 in 2024-05-15T15:28:22.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:22 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:28:23.362 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:23 smithi053 ceph-mon[25205]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:23.362 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:23 smithi053 ceph-mon[25205]: from='client.24236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:28:23.363 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:23 smithi053 ceph-mon[25205]: osdmap e35: 5 total, 5 up, 5 in 2024-05-15T15:28:23.363 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:28:23.363 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:23.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:23 smithi043 ceph-mon[24430]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:23.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:23 smithi043 ceph-mon[24430]: from='client.24236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:28:23.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:23 smithi043 ceph-mon[24430]: osdmap e35: 5 total, 5 up, 5 in 2024-05-15T15:28:23.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:28:23.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:23.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:23 smithi043 ceph-mon[29127]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:23.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:23 smithi043 ceph-mon[29127]: from='client.24236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:28:23.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:23 smithi043 ceph-mon[29127]: osdmap e35: 5 total, 5 up, 5 in 2024-05-15T15:28:23.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:28:23.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:24.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:28:24.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:28:24.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:24 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:28:24.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:24 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:28:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:24 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:28:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:24 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:28:25.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[24430]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-05-15T15:28:25.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/2319014455' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ab4ad5a-e5eb-46f2-a41f-82458b9b84af"}]: dispatch 2024-05-15T15:28:25.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ab4ad5a-e5eb-46f2-a41f-82458b9b84af"}]: dispatch 2024-05-15T15:28:25.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ab4ad5a-e5eb-46f2-a41f-82458b9b84af"}]': finished 2024-05-15T15:28:25.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[24430]: osdmap e36: 6 total, 5 up, 6 in 2024-05-15T15:28:25.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:25.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/4076463991' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:28:25.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[29127]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-05-15T15:28:25.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/2319014455' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ab4ad5a-e5eb-46f2-a41f-82458b9b84af"}]: dispatch 2024-05-15T15:28:25.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ab4ad5a-e5eb-46f2-a41f-82458b9b84af"}]: dispatch 2024-05-15T15:28:25.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ab4ad5a-e5eb-46f2-a41f-82458b9b84af"}]': finished 2024-05-15T15:28:25.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[29127]: osdmap e36: 6 total, 5 up, 6 in 2024-05-15T15:28:25.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:25.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:25 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/4076463991' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:28:25.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:25 smithi053 ceph-mon[25205]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-05-15T15:28:25.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:25 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/2319014455' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ab4ad5a-e5eb-46f2-a41f-82458b9b84af"}]: dispatch 2024-05-15T15:28:25.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:25 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ab4ad5a-e5eb-46f2-a41f-82458b9b84af"}]: dispatch 2024-05-15T15:28:25.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:25 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ab4ad5a-e5eb-46f2-a41f-82458b9b84af"}]': finished 2024-05-15T15:28:25.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:25 smithi053 ceph-mon[25205]: osdmap e36: 6 total, 5 up, 6 in 2024-05-15T15:28:25.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:25 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:25.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:25 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/4076463991' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:28:27.454 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:27 smithi053 ceph-mon[25205]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-05-15T15:28:27.585 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:27 smithi043 ceph-mon[29127]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-05-15T15:28:27.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:27 smithi043 ceph-mon[24430]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-05-15T15:28:29.263 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:29 smithi053 ceph-mon[25205]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-05-15T15:28:29.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:29 smithi043 ceph-mon[29127]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-05-15T15:28:29.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:29 smithi043 ceph-mon[24430]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-05-15T15:28:30.858 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:30 smithi053 ceph-mon[25205]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-15T15:28:30.858 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:30 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-15T15:28:30.858 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:30 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:30.858 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:30 smithi053 ceph-mon[25205]: Deploying daemon osd.5 on smithi053 2024-05-15T15:28:31.085 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:30 smithi043 ceph-mon[29127]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-15T15:28:31.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:30 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-15T15:28:31.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:30 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:31.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:30 smithi043 ceph-mon[29127]: Deploying daemon osd.5 on smithi053 2024-05-15T15:28:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:30 smithi043 ceph-mon[24430]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-15T15:28:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:30 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-15T15:28:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:30 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:30 smithi043 ceph-mon[24430]: Deploying daemon osd.5 on smithi053 2024-05-15T15:28:33.171 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:32 smithi053 ceph-mon[25205]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-05-15T15:28:33.335 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:32 smithi043 ceph-mon[29127]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-05-15T15:28:33.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:32 smithi043 ceph-mon[24430]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-05-15T15:28:35.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:34 smithi053 ceph-mon[25205]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:35.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:34 smithi043 ceph-mon[29127]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:35.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:34 smithi043 ceph-mon[24430]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:36.769 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:36 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:36.769 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:36 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:36.769 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:36 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:36.769 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:36 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:36.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:36 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:36.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:36 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:36.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:36 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:36.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:36 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:36.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:36.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:36.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:36.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:37.767 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:37 smithi053 ceph-mon[25205]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:37 smithi043 ceph-mon[29127]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:37.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:37 smithi043 ceph-mon[24430]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:38.465 INFO:teuthology.orchestra.run.smithi053.stdout:Created osd(s) 5 on host 'smithi053' 2024-05-15T15:28:39.044 DEBUG:teuthology.orchestra.run.smithi053:osd.5> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.5.service 2024-05-15T15:28:39.051 INFO:tasks.cephadm:Deploying osd.6 on smithi053 with /dev/vg_nvme/lv_2... 2024-05-15T15:28:39.051 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-15T15:28:39.553 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:39 smithi053 ceph-mon[25205]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:39.553 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:39.553 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:39 smithi053 ceph-mon[25205]: from='osd.5 [v2:172.21.15.53:6808/2835156943,v1:172.21.15.53:6809/2835156943]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T15:28:39.553 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:39 smithi053 ceph-mon[25205]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T15:28:39.553 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:39 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:39 smithi043 ceph-mon[29127]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:39 smithi043 ceph-mon[29127]: from='osd.5 [v2:172.21.15.53:6808/2835156943,v1:172.21.15.53:6809/2835156943]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T15:28:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:39 smithi043 ceph-mon[29127]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T15:28:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:39 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:39 smithi043 ceph-mon[24430]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:39 smithi043 ceph-mon[24430]: from='osd.5 [v2:172.21.15.53:6808/2835156943,v1:172.21.15.53:6809/2835156943]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T15:28:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:39 smithi043 ceph-mon[24430]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T15:28:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:39 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:40 smithi043 ceph-mon[24430]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-15T15:28:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:40 smithi043 ceph-mon[24430]: osdmap e37: 6 total, 5 up, 6 in 2024-05-15T15:28:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:40 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:40 smithi043 ceph-mon[24430]: from='osd.5 [v2:172.21.15.53:6808/2835156943,v1:172.21.15.53:6809/2835156943]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:40 smithi043 ceph-mon[24430]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:40 smithi043 ceph-mon[29127]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-15T15:28:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:40 smithi043 ceph-mon[29127]: osdmap e37: 6 total, 5 up, 6 in 2024-05-15T15:28:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:40 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:40.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:40 smithi043 ceph-mon[29127]: from='osd.5 [v2:172.21.15.53:6808/2835156943,v1:172.21.15.53:6809/2835156943]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:40.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:40 smithi043 ceph-mon[29127]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:40 smithi053 ceph-mon[25205]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-15T15:28:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:40 smithi053 ceph-mon[25205]: osdmap e37: 6 total, 5 up, 6 in 2024-05-15T15:28:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:40 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:40 smithi053 ceph-mon[25205]: from='osd.5 [v2:172.21.15.53:6808/2835156943,v1:172.21.15.53:6809/2835156943]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:40 smithi053 ceph-mon[25205]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:28:40.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 15:28:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[31415]: 2024-05-15T15:28:40.475+0000 7f31303c5700 -1 osd.5 0 waiting for initial osdmap 2024-05-15T15:28:40.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 15:28:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[31415]: 2024-05-15T15:28:40.480+0000 7f312a55b700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:28:41.764 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:41 smithi053 ceph-mon[25205]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:41.765 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:41 smithi053 ceph-mon[25205]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:28:41.765 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:41 smithi053 ceph-mon[25205]: osdmap e38: 6 total, 5 up, 6 in 2024-05-15T15:28:41.765 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:41 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:41.765 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:41 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:41 smithi043 ceph-mon[24430]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:41 smithi043 ceph-mon[24430]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:28:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:41 smithi043 ceph-mon[24430]: osdmap e38: 6 total, 5 up, 6 in 2024-05-15T15:28:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:41 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:41 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:41 smithi043 ceph-mon[29127]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-15T15:28:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:41 smithi043 ceph-mon[29127]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:28:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:41 smithi043 ceph-mon[29127]: osdmap e38: 6 total, 5 up, 6 in 2024-05-15T15:28:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:41 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:41.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:41 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:42.712 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:42 smithi053 ceph-mon[25205]: purged_snaps scrub starts 2024-05-15T15:28:42.713 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:42 smithi053 ceph-mon[25205]: purged_snaps scrub ok 2024-05-15T15:28:42.713 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:42 smithi053 ceph-mon[25205]: osd.5 [v2:172.21.15.53:6808/2835156943,v1:172.21.15.53:6809/2835156943] boot 2024-05-15T15:28:42.713 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:42 smithi053 ceph-mon[25205]: osdmap e39: 6 total, 6 up, 6 in 2024-05-15T15:28:42.713 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:42 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:42 smithi043 ceph-mon[24430]: purged_snaps scrub starts 2024-05-15T15:28:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:42 smithi043 ceph-mon[24430]: purged_snaps scrub ok 2024-05-15T15:28:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:42 smithi043 ceph-mon[24430]: osd.5 [v2:172.21.15.53:6808/2835156943,v1:172.21.15.53:6809/2835156943] boot 2024-05-15T15:28:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:42 smithi043 ceph-mon[24430]: osdmap e39: 6 total, 6 up, 6 in 2024-05-15T15:28:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:42 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:42 smithi043 ceph-mon[29127]: purged_snaps scrub starts 2024-05-15T15:28:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:42 smithi043 ceph-mon[29127]: purged_snaps scrub ok 2024-05-15T15:28:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:42 smithi043 ceph-mon[29127]: osd.5 [v2:172.21.15.53:6808/2835156943,v1:172.21.15.53:6809/2835156943] boot 2024-05-15T15:28:42.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:42 smithi043 ceph-mon[29127]: osdmap e39: 6 total, 6 up, 6 in 2024-05-15T15:28:42.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:42 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:28:42.837 INFO:teuthology.orchestra.run.smithi053.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-15T15:28:42.837 INFO:teuthology.orchestra.run.smithi053.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-15T15:28:42.838 INFO:teuthology.orchestra.run.smithi053.stderr: stderr: 10+0 records in 2024-05-15T15:28:42.838 INFO:teuthology.orchestra.run.smithi053.stderr:10+0 records out 2024-05-15T15:28:42.838 INFO:teuthology.orchestra.run.smithi053.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00952651 s, 1.1 GB/s 2024-05-15T15:28:42.838 INFO:teuthology.orchestra.run.smithi053.stderr:--> Zapping successful for: 2024-05-15T15:28:43.454 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch daemon add osd smithi053:vg_nvme/lv_2 2024-05-15T15:28:43.779 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:43.779 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: osdmap e40: 6 total, 6 up, 6 in 2024-05-15T15:28:43.779 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.779 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:28:43.779 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:28:43.779 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:43.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:43.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:43.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:43 smithi053 ceph-mon[25205]: osdmap e41: 6 total, 6 up, 6 in 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: osdmap e40: 6 total, 6 up, 6 in 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[24430]: osdmap e41: 6 total, 6 up, 6 in 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: osdmap e40: 6 total, 6 up, 6 in 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:43.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:43 smithi043 ceph-mon[29127]: osdmap e41: 6 total, 6 up, 6 in 2024-05-15T15:28:44.823 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:44 smithi053 ceph-mon[25205]: Detected new or changed devices on smithi053 2024-05-15T15:28:44.823 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:44 smithi053 ceph-mon[25205]: Adjusting osd_memory_target on smithi053 to 8546M 2024-05-15T15:28:44.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:44 smithi043 ceph-mon[24430]: Detected new or changed devices on smithi053 2024-05-15T15:28:44.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:44 smithi043 ceph-mon[24430]: Adjusting osd_memory_target on smithi053 to 8546M 2024-05-15T15:28:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:44 smithi043 ceph-mon[29127]: Detected new or changed devices on smithi053 2024-05-15T15:28:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:44 smithi043 ceph-mon[29127]: Adjusting osd_memory_target on smithi053 to 8546M 2024-05-15T15:28:45.729 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:45 smithi053 ceph-mon[25205]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:45.729 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:45 smithi053 ceph-mon[25205]: osdmap e42: 6 total, 6 up, 6 in 2024-05-15T15:28:45.729 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:45 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:28:45.730 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:45 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:28:45.730 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:45 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:45 smithi043 ceph-mon[24430]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:45 smithi043 ceph-mon[24430]: osdmap e42: 6 total, 6 up, 6 in 2024-05-15T15:28:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:45 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:28:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:45 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:28:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:45 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:45 smithi043 ceph-mon[29127]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:45 smithi043 ceph-mon[29127]: osdmap e42: 6 total, 6 up, 6 in 2024-05-15T15:28:45.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:45 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:28:45.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:45 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:28:45.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:45 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:46.827 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:46 smithi053 ceph-mon[25205]: from='client.24263 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:28:46.828 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:46 smithi053 ceph-mon[25205]: osdmap e43: 6 total, 6 up, 6 in 2024-05-15T15:28:46.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:46 smithi043 ceph-mon[24430]: from='client.24263 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:28:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:46 smithi043 ceph-mon[24430]: osdmap e43: 6 total, 6 up, 6 in 2024-05-15T15:28:46.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:46 smithi043 ceph-mon[29127]: from='client.24263 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:28:46.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:46 smithi043 ceph-mon[29127]: osdmap e43: 6 total, 6 up, 6 in 2024-05-15T15:28:47.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:47 smithi043 ceph-mon[24430]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:47 smithi043 ceph-mon[29127]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:47 smithi053 ceph-mon[25205]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/4287642519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11502520-c379-4b3d-a9d7-db332bd28cb6"}]: dispatch 2024-05-15T15:28:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11502520-c379-4b3d-a9d7-db332bd28cb6"}]: dispatch 2024-05-15T15:28:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "11502520-c379-4b3d-a9d7-db332bd28cb6"}]': finished 2024-05-15T15:28:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[24430]: osdmap e44: 7 total, 6 up, 7 in 2024-05-15T15:28:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:28:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/1808324349' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:28:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/4287642519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11502520-c379-4b3d-a9d7-db332bd28cb6"}]: dispatch 2024-05-15T15:28:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11502520-c379-4b3d-a9d7-db332bd28cb6"}]: dispatch 2024-05-15T15:28:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "11502520-c379-4b3d-a9d7-db332bd28cb6"}]': finished 2024-05-15T15:28:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[29127]: osdmap e44: 7 total, 6 up, 7 in 2024-05-15T15:28:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:28:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:48 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/1808324349' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:28:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:48 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/4287642519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11502520-c379-4b3d-a9d7-db332bd28cb6"}]: dispatch 2024-05-15T15:28:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:48 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11502520-c379-4b3d-a9d7-db332bd28cb6"}]: dispatch 2024-05-15T15:28:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:48 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "11502520-c379-4b3d-a9d7-db332bd28cb6"}]': finished 2024-05-15T15:28:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:48 smithi053 ceph-mon[25205]: osdmap e44: 7 total, 6 up, 7 in 2024-05-15T15:28:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:48 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:28:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:48 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/1808324349' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:28:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:49 smithi043 ceph-mon[24430]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:49 smithi043 ceph-mon[29127]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:49.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:49 smithi053 ceph-mon[25205]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:50.870 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:50 smithi053 ceph-mon[25205]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:51.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:50 smithi043 ceph-mon[24430]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:51.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:50 smithi043 ceph-mon[29127]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:53.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:52 smithi043 ceph-mon[24430]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:53.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:52 smithi043 ceph-mon[29127]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:53.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:52 smithi053 ceph-mon[25205]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:53 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-15T15:28:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:53 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:54.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:53 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-15T15:28:54.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:53 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:54.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:53 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-15T15:28:54.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:53 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:55.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:54 smithi053 ceph-mon[25205]: Deploying daemon osd.6 on smithi053 2024-05-15T15:28:55.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:54 smithi053 ceph-mon[25205]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:55.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:54 smithi043 ceph-mon[29127]: Deploying daemon osd.6 on smithi053 2024-05-15T15:28:55.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:54 smithi043 ceph-mon[29127]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:55.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:54 smithi043 ceph-mon[24430]: Deploying daemon osd.6 on smithi053 2024-05-15T15:28:55.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:54 smithi043 ceph-mon[24430]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:56 smithi043 ceph-mon[24430]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:56 smithi043 ceph-mon[29127]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:56 smithi053 ceph-mon[25205]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:59.047 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:58 smithi043 ceph-mon[29127]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:59.047 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:58 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:59.048 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:58 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:59.048 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:58 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:59.048 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:28:58 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:59.048 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:58 smithi043 ceph-mon[24430]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:59.048 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:58 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:59.048 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:58 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:59.048 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:58 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:59.048 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:28:58 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:28:59.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:58 smithi053 ceph-mon[25205]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:28:59.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:58 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:28:59.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:58 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:28:59.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:58 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:28:59.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:28:58 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:01.085 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:00 smithi043 ceph-mon[29127]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:29:01.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:00 smithi043 ceph-mon[24430]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:29:01.104 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:00 smithi053 ceph-mon[25205]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:29:01.396 INFO:teuthology.orchestra.run.smithi053.stdout:Created osd(s) 6 on host 'smithi053' 2024-05-15T15:29:02.465 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:02 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:02.465 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:02 smithi053 ceph-mon[25205]: from='osd.6 [v2:172.21.15.53:6816/2855453996,v1:172.21.15.53:6817/2855453996]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T15:29:02.465 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:02 smithi053 ceph-mon[25205]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T15:29:02.466 DEBUG:teuthology.orchestra.run.smithi053:osd.6> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.6.service 2024-05-15T15:29:02.468 INFO:tasks.cephadm:Deploying osd.7 on smithi053 with /dev/vg_nvme/lv_1... 2024-05-15T15:29:02.469 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-15T15:29:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:02 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:02 smithi043 ceph-mon[29127]: from='osd.6 [v2:172.21.15.53:6816/2855453996,v1:172.21.15.53:6817/2855453996]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T15:29:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:02 smithi043 ceph-mon[29127]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T15:29:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:02 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:02 smithi043 ceph-mon[24430]: from='osd.6 [v2:172.21.15.53:6816/2855453996,v1:172.21.15.53:6817/2855453996]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T15:29:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:02 smithi043 ceph-mon[24430]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T15:29:03.709 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:03 smithi053 ceph-mon[25205]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:29:03.709 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:03 smithi053 ceph-mon[25205]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-15T15:29:03.709 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:03 smithi053 ceph-mon[25205]: osdmap e45: 7 total, 6 up, 7 in 2024-05-15T15:29:03.709 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:03.710 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:03 smithi053 ceph-mon[25205]: from='osd.6 [v2:172.21.15.53:6816/2855453996,v1:172.21.15.53:6817/2855453996]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:03.710 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:03 smithi053 ceph-mon[25205]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:03.710 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:03 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:03.710 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 15:29:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[34296]: 2024-05-15T15:29:03.405+0000 7fde33ed6700 -1 osd.6 0 waiting for initial osdmap 2024-05-15T15:29:03.710 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 15:29:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[34296]: 2024-05-15T15:29:03.412+0000 7fde2e86d700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:29:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[29127]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:29:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[29127]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-15T15:29:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[29127]: osdmap e45: 7 total, 6 up, 7 in 2024-05-15T15:29:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[29127]: from='osd.6 [v2:172.21.15.53:6816/2855453996,v1:172.21.15.53:6817/2855453996]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[29127]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:03.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[24430]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:29:03.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[24430]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-15T15:29:03.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[24430]: osdmap e45: 7 total, 6 up, 7 in 2024-05-15T15:29:03.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:03.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[24430]: from='osd.6 [v2:172.21.15.53:6816/2855453996,v1:172.21.15.53:6817/2855453996]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:03.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[24430]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:03.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:03 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:04.502 INFO:teuthology.orchestra.run.smithi053.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-15T15:29:04.503 INFO:teuthology.orchestra.run.smithi053.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-15T15:29:04.503 INFO:teuthology.orchestra.run.smithi053.stderr: stderr: 10+0 records in 2024-05-15T15:29:04.503 INFO:teuthology.orchestra.run.smithi053.stderr:10+0 records out 2024-05-15T15:29:04.503 INFO:teuthology.orchestra.run.smithi053.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0100155 s, 1.0 GB/s 2024-05-15T15:29:04.503 INFO:teuthology.orchestra.run.smithi053.stderr:--> Zapping successful for: 2024-05-15T15:29:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:04 smithi053 ceph-mon[25205]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:29:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:04 smithi053 ceph-mon[25205]: osdmap e46: 7 total, 6 up, 7 in 2024-05-15T15:29:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:04 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:04 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:04 smithi043 ceph-mon[24430]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:29:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:04 smithi043 ceph-mon[24430]: osdmap e46: 7 total, 6 up, 7 in 2024-05-15T15:29:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:04 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:04 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:04 smithi043 ceph-mon[29127]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:29:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:04 smithi043 ceph-mon[29127]: osdmap e46: 7 total, 6 up, 7 in 2024-05-15T15:29:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:04 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:04 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:05.734 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:05 smithi053 ceph-mon[25205]: purged_snaps scrub starts 2024-05-15T15:29:05.734 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:05 smithi053 ceph-mon[25205]: purged_snaps scrub ok 2024-05-15T15:29:05.734 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:05 smithi053 ceph-mon[25205]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:29:05.734 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:05 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:05.734 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:05 smithi053 ceph-mon[25205]: osd.6 [v2:172.21.15.53:6816/2855453996,v1:172.21.15.53:6817/2855453996] boot 2024-05-15T15:29:05.734 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:05 smithi053 ceph-mon[25205]: osdmap e47: 7 total, 7 up, 7 in 2024-05-15T15:29:05.734 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:05 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:05.734 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:05 smithi053 ceph-mon[25205]: osdmap e48: 7 total, 7 up, 7 in 2024-05-15T15:29:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[24430]: purged_snaps scrub starts 2024-05-15T15:29:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[24430]: purged_snaps scrub ok 2024-05-15T15:29:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[24430]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:29:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[24430]: osd.6 [v2:172.21.15.53:6816/2855453996,v1:172.21.15.53:6817/2855453996] boot 2024-05-15T15:29:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[24430]: osdmap e47: 7 total, 7 up, 7 in 2024-05-15T15:29:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[24430]: osdmap e48: 7 total, 7 up, 7 in 2024-05-15T15:29:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[29127]: purged_snaps scrub starts 2024-05-15T15:29:05.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[29127]: purged_snaps scrub ok 2024-05-15T15:29:05.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[29127]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-15T15:29:05.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:05.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[29127]: osd.6 [v2:172.21.15.53:6816/2855453996,v1:172.21.15.53:6817/2855453996] boot 2024-05-15T15:29:05.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[29127]: osdmap e47: 7 total, 7 up, 7 in 2024-05-15T15:29:05.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:05.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:05 smithi043 ceph-mon[29127]: osdmap e48: 7 total, 7 up, 7 in 2024-05-15T15:29:06.218 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch daemon add osd smithi053:vg_nvme/lv_1 2024-05-15T15:29:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:06 smithi043 ceph-mon[24430]: osdmap e49: 7 total, 7 up, 7 in 2024-05-15T15:29:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:06 smithi043 ceph-mon[24430]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:06 smithi043 ceph-mon[29127]: osdmap e49: 7 total, 7 up, 7 in 2024-05-15T15:29:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:06 smithi043 ceph-mon[29127]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:06 smithi053 ceph-mon[25205]: osdmap e49: 7 total, 7 up, 7 in 2024-05-15T15:29:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:06 smithi053 ceph-mon[25205]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:07.828 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:07 smithi053 ceph-mon[25205]: osdmap e50: 7 total, 7 up, 7 in 2024-05-15T15:29:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:07 smithi043 ceph-mon[24430]: osdmap e50: 7 total, 7 up, 7 in 2024-05-15T15:29:08.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:07 smithi043 ceph-mon[29127]: osdmap e50: 7 total, 7 up, 7 in 2024-05-15T15:29:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: osdmap e51: 7 total, 7 up, 7 in 2024-05-15T15:29:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: Detected new or changed devices on smithi053 2024-05-15T15:29:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:08.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:08.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:08.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: Adjusting osd_memory_target on smithi053 to 5697M 2024-05-15T15:29:08.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:08.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:08.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:08.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:08.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:08.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:08 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: osdmap e51: 7 total, 7 up, 7 in 2024-05-15T15:29:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: Detected new or changed devices on smithi053 2024-05-15T15:29:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: Adjusting osd_memory_target on smithi053 to 5697M 2024-05-15T15:29:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:09.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:09.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:09.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:09.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:09.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:09.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: osdmap e51: 7 total, 7 up, 7 in 2024-05-15T15:29:09.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: Detected new or changed devices on smithi053 2024-05-15T15:29:09.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: Adjusting osd_memory_target on smithi053 to 5697M 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:09.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:08 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:09 smithi053 ceph-mon[25205]: from='client.24290 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:09 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:29:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:09 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:29:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:09 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:10.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:09 smithi043 ceph-mon[24430]: from='client.24290 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:10.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:09 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:29:10.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:09 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:29:10.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:09 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:10.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:09 smithi043 ceph-mon[29127]: from='client.24290 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:10.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:09 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-15T15:29:10.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:09 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-15T15:29:10.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:09 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:10 smithi053 ceph-mon[25205]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:11.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:10 smithi043 ceph-mon[24430]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:11.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:10 smithi043 ceph-mon[29127]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:11 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/837419647' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f9ad71-027e-4836-8508-662d813d3f0c"}]: dispatch 2024-05-15T15:29:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:11 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f9ad71-027e-4836-8508-662d813d3f0c"}]: dispatch 2024-05-15T15:29:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:11 smithi053 ceph-mon[25205]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12f9ad71-027e-4836-8508-662d813d3f0c"}]': finished 2024-05-15T15:29:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:11 smithi053 ceph-mon[25205]: osdmap e52: 8 total, 7 up, 8 in 2024-05-15T15:29:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:11 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:11 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/990297868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:29:12.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/837419647' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f9ad71-027e-4836-8508-662d813d3f0c"}]: dispatch 2024-05-15T15:29:12.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f9ad71-027e-4836-8508-662d813d3f0c"}]: dispatch 2024-05-15T15:29:12.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[24430]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12f9ad71-027e-4836-8508-662d813d3f0c"}]': finished 2024-05-15T15:29:12.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[24430]: osdmap e52: 8 total, 7 up, 8 in 2024-05-15T15:29:12.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:12.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/990297868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:29:12.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/837419647' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f9ad71-027e-4836-8508-662d813d3f0c"}]: dispatch 2024-05-15T15:29:12.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f9ad71-027e-4836-8508-662d813d3f0c"}]: dispatch 2024-05-15T15:29:12.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[29127]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12f9ad71-027e-4836-8508-662d813d3f0c"}]': finished 2024-05-15T15:29:12.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[29127]: osdmap e52: 8 total, 7 up, 8 in 2024-05-15T15:29:12.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:12.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:11 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/990297868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-15T15:29:13.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:13 smithi043 ceph-mon[24430]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-15T15:29:13.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:13 smithi043 ceph-mon[29127]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-15T15:29:13.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:13 smithi053 ceph-mon[25205]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-15T15:29:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:15 smithi053 ceph-mon[25205]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-15T15:29:15.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:15 smithi043 ceph-mon[24430]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-15T15:29:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:15 smithi043 ceph-mon[29127]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-15T15:29:17.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:17 smithi043 ceph-mon[24430]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-05-15T15:29:17.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:17 smithi043 ceph-mon[29127]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-05-15T15:29:17.613 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:17 smithi053 ceph-mon[25205]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-05-15T15:29:18.333 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:18 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-15T15:29:18.333 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:18 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:18.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:18 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-15T15:29:18.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:18 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:18.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:18 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-15T15:29:18.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:18 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:19 smithi053 ceph-mon[25205]: Deploying daemon osd.7 on smithi053 2024-05-15T15:29:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:19 smithi053 ceph-mon[25205]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-15T15:29:19.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:19 smithi043 ceph-mon[24430]: Deploying daemon osd.7 on smithi053 2024-05-15T15:29:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:19 smithi043 ceph-mon[24430]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-15T15:29:19.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:19 smithi043 ceph-mon[29127]: Deploying daemon osd.7 on smithi053 2024-05-15T15:29:19.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:19 smithi043 ceph-mon[29127]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-15T15:29:21.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:20 smithi043 ceph-mon[24430]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-15T15:29:21.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:20 smithi043 ceph-mon[29127]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-15T15:29:21.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:20 smithi053 ceph-mon[25205]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-15T15:29:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:23 smithi043 ceph-mon[24430]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:23 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:23 smithi043 ceph-mon[29127]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:23 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:23 smithi053 ceph-mon[25205]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:23 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:24 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:29:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:24 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:29:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:29:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:24 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:29:24.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:24 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:29:24.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:24 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:29:25.738 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:25 smithi053 ceph-mon[25205]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:25.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:25 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:25.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:25 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:25 smithi043 ceph-mon[24430]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:25 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:25 smithi043 ceph-mon[29127]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:25 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:25 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:26.024 INFO:teuthology.orchestra.run.smithi053.stdout:Created osd(s) 7 on host 'smithi053' 2024-05-15T15:29:26.609 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:26 smithi053 ceph-mon[25205]: from='osd.7 [v2:172.21.15.53:6824/3221065522,v1:172.21.15.53:6825/3221065522]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T15:29:26.609 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:26 smithi053 ceph-mon[25205]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T15:29:26.609 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:26 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:26.609 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:26 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:26.609 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:26 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:26.609 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:26 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:26.632 DEBUG:teuthology.orchestra.run.smithi053:osd.7> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.7.service 2024-05-15T15:29:26.634 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-05-15T15:29:26.634 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd stat -f json 2024-05-15T15:29:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[24430]: from='osd.7 [v2:172.21.15.53:6824/3221065522,v1:172.21.15.53:6825/3221065522]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T15:29:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[24430]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T15:29:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[29127]: from='osd.7 [v2:172.21.15.53:6824/3221065522,v1:172.21.15.53:6825/3221065522]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T15:29:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[29127]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T15:29:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:26 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:27.682 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:27 smithi053 ceph-mon[25205]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:27.683 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:27 smithi053 ceph-mon[25205]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-15T15:29:27.683 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:27 smithi053 ceph-mon[25205]: osdmap e53: 8 total, 7 up, 8 in 2024-05-15T15:29:27.683 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:27 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:27.683 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:27 smithi053 ceph-mon[25205]: from='osd.7 [v2:172.21.15.53:6824/3221065522,v1:172.21.15.53:6825/3221065522]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:27.683 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:27 smithi053 ceph-mon[25205]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:27.683 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 15:29:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[37268]: 2024-05-15T15:29:27.557+0000 7fa053946700 -1 osd.7 0 waiting for initial osdmap 2024-05-15T15:29:27.683 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 15:29:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[37268]: 2024-05-15T15:29:27.567+0000 7fa04eade700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:29:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[24430]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[24430]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-15T15:29:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[24430]: osdmap e53: 8 total, 7 up, 8 in 2024-05-15T15:29:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[24430]: from='osd.7 [v2:172.21.15.53:6824/3221065522,v1:172.21.15.53:6825/3221065522]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[24430]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:27.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[29127]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:27.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[29127]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-15T15:29:27.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[29127]: osdmap e53: 8 total, 7 up, 8 in 2024-05-15T15:29:27.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:27.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[29127]: from='osd.7 [v2:172.21.15.53:6824/3221065522,v1:172.21.15.53:6825/3221065522]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:27.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:27 smithi043 ceph-mon[29127]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T15:29:28.803 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:29:28.803 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:28 smithi043 ceph-mon[24430]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:29:28.803 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:28 smithi043 ceph-mon[24430]: osdmap e54: 8 total, 7 up, 8 in 2024-05-15T15:29:28.803 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:28 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:28.804 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:28 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:28.804 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:28 smithi043 ceph-mon[29127]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:29:28.804 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:28 smithi043 ceph-mon[29127]: osdmap e54: 8 total, 7 up, 8 in 2024-05-15T15:29:28.804 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:28 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:28.804 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:28 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:28 smithi053 ceph-mon[25205]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-05-15T15:29:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:28 smithi053 ceph-mon[25205]: osdmap e54: 8 total, 7 up, 8 in 2024-05-15T15:29:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:28 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:28 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:29.494 INFO:teuthology.orchestra.run.smithi043.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1715786968,"num_in_osds":8,"osd_in_since":1715786951,"num_remapped_pgs":1} 2024-05-15T15:29:29.495 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd dump --format=json 2024-05-15T15:29:29.730 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:29:29.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: purged_snaps scrub starts 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: purged_snaps scrub ok 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: osd.7 [v2:172.21.15.53:6824/3221065522,v1:172.21.15.53:6825/3221065522] boot 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: osdmap e55: 8 total, 8 up, 8 in 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2179605081' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: Detected new or changed devices on smithi053 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: Adjusting osd_memory_target on smithi053 to 4273M 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: purged_snaps scrub starts 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: purged_snaps scrub ok 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: osd.7 [v2:172.21.15.53:6824/3221065522,v1:172.21.15.53:6825/3221065522] boot 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: osdmap e55: 8 total, 8 up, 8 in 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2179605081' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: Detected new or changed devices on smithi053 2024-05-15T15:29:29.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:29.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: Adjusting osd_memory_target on smithi053 to 4273M 2024-05-15T15:29:29.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:29.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:29 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: purged_snaps scrub starts 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: purged_snaps scrub ok 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: osd.7 [v2:172.21.15.53:6824/3221065522,v1:172.21.15.53:6825/3221065522] boot 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: osdmap e55: 8 total, 8 up, 8 in 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2179605081' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: Detected new or changed devices on smithi053 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:29:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: Adjusting osd_memory_target on smithi053 to 4273M 2024-05-15T15:29:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:29 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:30.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:30 smithi043 ceph-mon[29127]: osdmap e56: 8 total, 8 up, 8 in 2024-05-15T15:29:30.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:30 smithi043 ceph-mon[29127]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:30.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:30 smithi043 ceph-mon[29127]: osdmap e57: 8 total, 8 up, 8 in 2024-05-15T15:29:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:30 smithi043 ceph-mon[24430]: osdmap e56: 8 total, 8 up, 8 in 2024-05-15T15:29:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:30 smithi043 ceph-mon[24430]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:30 smithi043 ceph-mon[24430]: osdmap e57: 8 total, 8 up, 8 in 2024-05-15T15:29:30.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:30 smithi053 ceph-mon[25205]: osdmap e56: 8 total, 8 up, 8 in 2024-05-15T15:29:30.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:30 smithi053 ceph-mon[25205]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:30.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:30 smithi053 ceph-mon[25205]: osdmap e57: 8 total, 8 up, 8 in 2024-05-15T15:29:31.640 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:29:31.640 INFO:teuthology.orchestra.run.smithi043.stdout:{"epoch":58,"fsid":"22d82864-12cf-11ef-bc98-c7b262605968","created":"2024-05-15T15:24:28.633262+0000","modified":"2024-05-15T15:29:31.567201+0000","last_up_change":"2024-05-15T15:29:28.553820+0000","last_in_change":"2024-05-15T15:29:11.147309+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-15T15:27:32.341598+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7ac451f1-36bd-4ff8-8b90-92a83e712159","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6802","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6803","nonce":2576871983}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6804","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6805","nonce":2576871983}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6808","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6809","nonce":2576871983}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6806","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6807","nonce":2576871983}]},"public_addr":"172.21.15.43:6803/2576871983","cluster_addr":"172.21.15.43:6805/2576871983","heartbeat_back_addr":"172.21.15.43:6809/2576871983","heartbeat_front_addr":"172.21.15.43:6807/2576871983","state":["exists","up"]},{"osd":1,"uuid":"c35bd4a6-ea35-419f-a5cc-fc97f9eb509d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6810","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6811","nonce":3463535224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6812","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6813","nonce":3463535224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6816","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6817","nonce":3463535224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6814","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6815","nonce":3463535224}]},"public_addr":"172.21.15.43:6811/3463535224","cluster_addr":"172.21.15.43:6813/3463535224","heartbeat_back_addr":"172.21.15.43:6817/3463535224","heartbeat_front_addr":"172.21.15.43:6815/3463535224","state":["exists","up"]},{"osd":2,"uuid":"7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6818","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6819","nonce":50306098}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6820","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6821","nonce":50306098}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6824","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6825","nonce":50306098}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6822","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6823","nonce":50306098}]},"public_addr":"172.21.15.43:6819/50306098","cluster_addr":"172.21.15.43:6821/50306098","heartbeat_back_addr":"172.21.15.43:6825/50306098","heartbeat_front_addr":"172.21.15.43:6823/50306098","state":["exists","up"]},{"osd":3,"uuid":"f79328f3-261d-4a48-8a97-952a3bf9b97e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6826","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6827","nonce":3944497486}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6828","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6829","nonce":3944497486}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6832","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6833","nonce":3944497486}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6830","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6831","nonce":3944497486}]},"public_addr":"172.21.15.43:6827/3944497486","cluster_addr":"172.21.15.43:6829/3944497486","heartbeat_back_addr":"172.21.15.43:6833/3944497486","heartbeat_front_addr":"172.21.15.43:6831/3944497486","state":["exists","up"]},{"osd":4,"uuid":"50ed214e-19e1-4bab-95ac-d636276f8925","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6800","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6801","nonce":315819048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6802","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6803","nonce":315819048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6806","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6807","nonce":315819048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6804","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6805","nonce":315819048}]},"public_addr":"172.21.15.53:6801/315819048","cluster_addr":"172.21.15.53:6803/315819048","heartbeat_back_addr":"172.21.15.53:6807/315819048","heartbeat_front_addr":"172.21.15.53:6805/315819048","state":["exists","up"]},{"osd":5,"uuid":"4ab4ad5a-e5eb-46f2-a41f-82458b9b84af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6808","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6809","nonce":2835156943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6810","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6811","nonce":2835156943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6814","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6815","nonce":2835156943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6812","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6813","nonce":2835156943}]},"public_addr":"172.21.15.53:6809/2835156943","cluster_addr":"172.21.15.53:6811/2835156943","heartbeat_back_addr":"172.21.15.53:6815/2835156943","heartbeat_front_addr":"172.21.15.53:6813/2835156943","state":["exists","up"]},{"osd":6,"uuid":"11502520-c379-4b3d-a9d7-db332bd28cb6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6816","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6817","nonce":2855453996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6818","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6819","nonce":2855453996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6822","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6823","nonce":2855453996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6820","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6821","nonce":2855453996}]},"public_addr":"172.21.15.53:6817/2855453996","cluster_addr":"172.21.15.53:6819/2855453996","heartbeat_back_addr":"172.21.15.53:6823/2855453996","heartbeat_front_addr":"172.21.15.53:6821/2855453996","state":["exists","up"]},{"osd":7,"uuid":"12f9ad71-027e-4836-8508-662d813d3f0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6824","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6825","nonce":3221065522}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6826","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6827","nonce":3221065522}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6830","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6831","nonce":3221065522}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6828","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6829","nonce":3221065522}]},"public_addr":"172.21.15.53:6825/3221065522","cluster_addr":"172.21.15.53:6827/3221065522","heartbeat_back_addr":"172.21.15.53:6831/3221065522","heartbeat_front_addr":"172.21.15.53:6829/3221065522","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:26:41.970916+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:27:05.203106+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:27:28.889033+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:27:54.138937+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:28:16.062057+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:28:39.548674+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:29:02.818322+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:29:26.663626+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.43:0/3975126779":"2024-05-16T15:25:23.773919+0000","172.21.15.43:6800/970314706":"2024-05-16T15:25:23.773919+0000","172.21.15.43:0/1710373736":"2024-05-16T15:25:23.773919+0000","172.21.15.43:0/573610528":"2024-05-16T15:25:23.773919+0000","172.21.15.43:6800/2411931462":"2024-05-16T15:25:01.016490+0000","172.21.15.43:6801/2411931462":"2024-05-16T15:25:01.016490+0000","172.21.15.43:0/653624395":"2024-05-16T15:25:01.016490+0000","172.21.15.43:0/1103755388":"2024-05-16T15:25:01.016490+0000","172.21.15.43:6801/970314706":"2024-05-16T15:25:23.773919+0000","172.21.15.43:0/3154897789":"2024-05-16T15:25:01.016490+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-15T15:29:32.373 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-05-15T15:27:32.341598+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-05-15T15:29:32.374 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-05-15T15:29:32.607 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:29:32.727 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:32 smithi043 ceph-mon[24430]: osdmap e58: 8 total, 8 up, 8 in 2024-05-15T15:29:32.727 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:32 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3492296421' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-15T15:29:32.727 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:32 smithi043 ceph-mon[24430]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:32.727 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:32 smithi043 ceph-mon[29127]: osdmap e58: 8 total, 8 up, 8 in 2024-05-15T15:29:32.727 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:32 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3492296421' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-15T15:29:32.727 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:32 smithi043 ceph-mon[29127]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:32.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:32 smithi053 ceph-mon[25205]: osdmap e58: 8 total, 8 up, 8 in 2024-05-15T15:29:32.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:32 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3492296421' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-15T15:29:32.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:32 smithi053 ceph-mon[25205]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:33.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:33 smithi043 ceph-mon[24430]: osdmap e59: 8 total, 8 up, 8 in 2024-05-15T15:29:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:33 smithi043 ceph-mon[29127]: osdmap e59: 8 total, 8 up, 8 in 2024-05-15T15:29:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:33 smithi053 ceph-mon[25205]: osdmap e59: 8 total, 8 up, 8 in 2024-05-15T15:29:34.262 INFO:teuthology.orchestra.run.smithi043.stdout:pg_num: 1 2024-05-15T15:29:34.830 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:34 smithi043 ceph-mon[24430]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-05-15T15:29:34.831 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:34 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1459545924' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-15T15:29:34.831 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:34 smithi043 ceph-mon[29127]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-05-15T15:29:34.832 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:34 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1459545924' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-15T15:29:34.858 INFO:tasks.cephadm:Adding prometheus.a on smithi053 2024-05-15T15:29:34.859 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch apply prometheus '1;smithi053=a' 2024-05-15T15:29:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:34 smithi053 ceph-mon[25205]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-05-15T15:29:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:34 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1459545924' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-15T15:29:36.824 INFO:teuthology.orchestra.run.smithi053.stdout:Scheduled prometheus update... 2024-05-15T15:29:37.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:36 smithi053 ceph-mon[25205]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-15T15:29:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:36 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:36 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:36 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:36 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:36 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:36 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-15T15:29:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[24430]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-15T15:29:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-15T15:29:37.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[29127]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-15T15:29:37.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:37.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:37.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:37.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:37.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' 2024-05-15T15:29:37.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:36 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-15T15:29:37.451 DEBUG:teuthology.orchestra.run.smithi053:prometheus.a> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service 2024-05-15T15:29:37.495 INFO:tasks.cephadm:Adding node-exporter.a on smithi043 2024-05-15T15:29:37.495 INFO:tasks.cephadm:Adding node-exporter.b on smithi053 2024-05-15T15:29:37.495 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch apply node-exporter '2;smithi043=a;smithi053=b' 2024-05-15T15:29:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:37 smithi053 ceph-mon[25205]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi053=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:38.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:37 smithi053 ceph-mon[25205]: Saving service prometheus spec with placement smithi053=a;count:1 2024-05-15T15:29:38.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:37 smithi053 ceph-mon[25205]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-15T15:29:38.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:37 smithi053 ceph-mon[25205]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-15T15:29:38.131 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ignoring --setuser ceph since I am not root 2024-05-15T15:29:38.131 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ignoring --setgroup ceph since I am not root 2024-05-15T15:29:38.131 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:37 smithi053 ceph-mgr[26318]: -- 172.21.15.53:0/3919701272 <== mon.2 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55983b9fc340 con 0x55983b9d5000 2024-05-15T15:29:38.132 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:38.043+0000 7ff04415d000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:29:38.190 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:37 smithi043 ceph-mon[29127]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi053=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:38.190 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:37 smithi043 ceph-mon[29127]: Saving service prometheus spec with placement smithi053=a;count:1 2024-05-15T15:29:38.190 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:37 smithi043 ceph-mon[29127]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-15T15:29:38.190 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:37 smithi043 ceph-mon[29127]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-15T15:29:38.191 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:37 smithi043 ceph-mon[24430]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi053=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:38.191 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:37 smithi043 ceph-mon[24430]: Saving service prometheus spec with placement smithi053=a;count:1 2024-05-15T15:29:38.191 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:37 smithi043 ceph-mon[24430]: from='mgr.14150 172.21.15.43:0/1916769803' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-15T15:29:38.191 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:37 smithi043 ceph-mon[24430]: mgrmap e16: y(active, since 4m), standbys: x 2024-05-15T15:29:38.191 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ignoring --setuser ceph since I am not root 2024-05-15T15:29:38.191 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ignoring --setgroup ceph since I am not root 2024-05-15T15:29:38.191 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:37 smithi043 ceph-mgr[24660]: -- 172.21.15.43:0/1537071323 <== mon.0 v2:172.21.15.43:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55706193a340 con 0x557061912c00 2024-05-15T15:29:38.191 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:38 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:38.055+0000 7f6232046000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:29:38.191 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:38 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:38.188+0000 7f6232046000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:29:38.501 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:38.180+0000 7ff04415d000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:29:38.501 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:38.283+0000 7ff04415d000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:29:38.585 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:38 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:38.290+0000 7f6232046000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:29:38.810 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:38.604+0000 7ff04415d000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:29:38.810 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:38.691+0000 7ff04415d000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:29:39.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:38 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:38.607+0000 7f6232046000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:29:39.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:38 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:38.693+0000 7f6232046000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:29:40.130 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:39.872+0000 7f6232046000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:29:40.130 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:40.043+0000 7f6232046000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:29:40.131 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:40.128+0000 7f6232046000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:29:40.153 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:39 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:39.895+0000 7ff04415d000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:29:40.153 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:40.066+0000 7ff04415d000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:29:40.153 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:40.151+0000 7ff04415d000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:29:40.808 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:40.458+0000 7f6232046000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:29:40.809 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:40.552+0000 7f6232046000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:29:40.809 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:40.648+0000 7f6232046000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:29:40.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:40.478+0000 7ff04415d000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:29:40.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:40.573+0000 7ff04415d000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:29:40.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:40.669+0000 7ff04415d000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:29:41.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:40.902+0000 7f6232046000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:29:41.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:40.920+0000 7ff04415d000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:29:41.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:41 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:41.363+0000 7f6232046000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:29:41.879 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:41 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:41.384+0000 7ff04415d000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:29:42.280 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:41 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:41.871+0000 7f6232046000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:29:42.280 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:41 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:41.969+0000 7f6232046000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:29:42.280 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:42.278+0000 7f6232046000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:29:42.298 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:41 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:41.890+0000 7ff04415d000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:29:42.298 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:41 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:41.988+0000 7ff04415d000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:29:42.585 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:42.372+0000 7f6232046000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:29:42.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:42.296+0000 7ff04415d000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:29:42.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:42.389+0000 7ff04415d000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:29:43.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:42.795+0000 7f6232046000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:29:43.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:42.934+0000 7f6232046000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:29:43.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:43.032+0000 7f6232046000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:29:43.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:42.810+0000 7ff04415d000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:29:43.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:42.948+0000 7ff04415d000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:29:43.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:43.044+0000 7ff04415d000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:29:43.585 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:43.152+0000 7f6232046000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:29:43.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:43.245+0000 7f6232046000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:29:43.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:43.162+0000 7ff04415d000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:29:43.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:43.254+0000 7ff04415d000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:29:44.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:43.819+0000 7f6232046000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:29:44.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:43.828+0000 7ff04415d000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:29:44.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[29127]: Active manager daemon y restarted 2024-05-15T15:29:44.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[29127]: Activating manager daemon y 2024-05-15T15:29:44.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[29127]: osdmap e60: 8 total, 8 up, 8 in 2024-05-15T15:29:44.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[29127]: Standby manager daemon x restarted 2024-05-15T15:29:44.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[29127]: Standby manager daemon x started 2024-05-15T15:29:44.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:29:44.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:29:44.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:29:44.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:29:44.504 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[24430]: Active manager daemon y restarted 2024-05-15T15:29:44.504 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[24430]: Activating manager daemon y 2024-05-15T15:29:44.504 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[24430]: osdmap e60: 8 total, 8 up, 8 in 2024-05-15T15:29:44.504 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[24430]: Standby manager daemon x restarted 2024-05-15T15:29:44.504 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[24430]: Standby manager daemon x started 2024-05-15T15:29:44.504 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:29:44.504 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:29:44.504 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:29:44.504 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:44 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:29:44.505 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:44.196+0000 7f6232046000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:29:44.505 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:29:44] ENGINE Bus STARTING 2024-05-15T15:29:44.505 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: CherryPy Checker: 2024-05-15T15:29:44.505 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: The Application mounted at '' has an empty config. 2024-05-15T15:29:44.505 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:29:44.505 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:29:44] ENGINE Serving on http://:::9283 2024-05-15T15:29:44.505 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:29:44] ENGINE Bus STARTED 2024-05-15T15:29:44.554 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:44 smithi053 ceph-mon[25205]: Active manager daemon y restarted 2024-05-15T15:29:44.554 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:44 smithi053 ceph-mon[25205]: Activating manager daemon y 2024-05-15T15:29:44.554 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:44 smithi053 ceph-mon[25205]: osdmap e60: 8 total, 8 up, 8 in 2024-05-15T15:29:44.554 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:44 smithi053 ceph-mon[25205]: Standby manager daemon x restarted 2024-05-15T15:29:44.554 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:44 smithi053 ceph-mon[25205]: Standby manager daemon x started 2024-05-15T15:29:44.554 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:44 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:29:44.554 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:44 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:29:44.554 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:44 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:29:44.555 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:44 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/962408588' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:29:44.555 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:44.201+0000 7ff04415d000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:29:44.555 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: [15/May/2024:15:29:44] ENGINE Bus STARTING 2024-05-15T15:29:44.555 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: CherryPy Checker: 2024-05-15T15:29:44.555 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: The Application mounted at '' has an empty config. 2024-05-15T15:29:44.555 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: 2024-05-15T15:29:44.555 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: [15/May/2024:15:29:44] ENGINE Serving on http://:::9283 2024-05-15T15:29:44.555 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:29:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: [15/May/2024:15:29:44] ENGINE Bus STARTED 2024-05-15T15:29:45.279 INFO:teuthology.orchestra.run.smithi053.stdout:Scheduled node-exporter update... 2024-05-15T15:29:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: mgrmap e17: y(active, starting, since 0.0556365s), standbys: x 2024-05-15T15:29:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:29:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:29:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:29:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:29:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:29:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:29:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: Manager daemon y is now available 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:29:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:29:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:29:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:45 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:29:45.399 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:29:45] ENGINE Bus STARTING 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: mgrmap e17: y(active, starting, since 0.0556365s), standbys: x 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:29:45.400 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: Manager daemon y is now available 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:29:45.401 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: mgrmap e17: y(active, starting, since 0.0556365s), standbys: x 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:29:45.402 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: Manager daemon y is now available 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:29:45.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:45 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:29:45.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:29:45] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:29:45.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:29:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:29:45] ENGINE Bus STARTED 2024-05-15T15:29:46.108 DEBUG:teuthology.orchestra.run.smithi043:node-exporter.a> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.a.service 2024-05-15T15:29:46.111 DEBUG:teuthology.orchestra.run.smithi053:node-exporter.b> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.b.service 2024-05-15T15:29:46.114 INFO:tasks.cephadm:Adding alertmanager.a on smithi043 2024-05-15T15:29:46.114 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch apply alertmanager '1;smithi043=a' 2024-05-15T15:29:46.351 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:46 smithi053 ceph-mon[25205]: mgrmap e18: y(active, since 1.06547s), standbys: x 2024-05-15T15:29:46.351 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:46 smithi053 ceph-mon[25205]: Saving service node-exporter spec with placement smithi043=a;smithi053=b;count:2 2024-05-15T15:29:46.351 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:46 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:46.351 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:46 smithi053 ceph-mon[25205]: [15/May/2024:15:29:45] ENGINE Bus STARTING 2024-05-15T15:29:46.351 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:46 smithi053 ceph-mon[25205]: [15/May/2024:15:29:45] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:29:46.351 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:46 smithi053 ceph-mon[25205]: [15/May/2024:15:29:45] ENGINE Bus STARTED 2024-05-15T15:29:46.352 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:46 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:46.352 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:46 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:46.483 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[24430]: mgrmap e18: y(active, since 1.06547s), standbys: x 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[24430]: Saving service node-exporter spec with placement smithi043=a;smithi053=b;count:2 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[24430]: [15/May/2024:15:29:45] ENGINE Bus STARTING 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[24430]: [15/May/2024:15:29:45] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[24430]: [15/May/2024:15:29:45] ENGINE Bus STARTED 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[29127]: mgrmap e18: y(active, since 1.06547s), standbys: x 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[29127]: Saving service node-exporter spec with placement smithi043=a;smithi053=b;count:2 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:46.484 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[29127]: [15/May/2024:15:29:45] ENGINE Bus STARTING 2024-05-15T15:29:46.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[29127]: [15/May/2024:15:29:45] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:29:46.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[29127]: [15/May/2024:15:29:45] ENGINE Bus STARTED 2024-05-15T15:29:46.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:46.485 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:46 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:47 smithi053 ceph-mon[25205]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:47.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:47 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:47 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:47 smithi053 ceph-mon[25205]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:29:47.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:47 smithi053 ceph-mon[25205]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:29:47.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:47 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:47 smithi053 ceph-mon[25205]: mgrmap e19: y(active, since 2s), standbys: x 2024-05-15T15:29:47.360 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:47 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[29127]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:29:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:29:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[29127]: mgrmap e19: y(active, since 2s), standbys: x 2024-05-15T15:29:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[24430]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:47.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:29:47.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:29:47.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:47.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[24430]: mgrmap e19: y(active, since 2s), standbys: x 2024-05-15T15:29:47.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:47 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:48.273 INFO:teuthology.orchestra.run.smithi053.stdout:Scheduled alertmanager update... 2024-05-15T15:29:48.604 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:48 smithi053 ceph-mon[25205]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:29:48.604 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:48 smithi053 ceph-mon[25205]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:29:48.604 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:48 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:48.604 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:48 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:48.604 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:48 smithi053 ceph-mon[25205]: Deploying daemon prometheus.a on smithi053 2024-05-15T15:29:48.604 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:48 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:29:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:29:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[24430]: Deploying daemon prometheus.a on smithi053 2024-05-15T15:29:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:29:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:29:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[29127]: Deploying daemon prometheus.a on smithi053 2024-05-15T15:29:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:48 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:48.897 DEBUG:teuthology.orchestra.run.smithi043:alertmanager.a> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@alertmanager.a.service 2024-05-15T15:29:48.900 INFO:tasks.cephadm:Adding grafana.a on smithi053 2024-05-15T15:29:48.900 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph orch apply grafana '1;smithi053=a' 2024-05-15T15:29:49.831 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:49 smithi053 ceph-mon[25205]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:49.831 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:49 smithi053 ceph-mon[25205]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi043=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:49.832 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:49 smithi053 ceph-mon[25205]: Saving service alertmanager spec with placement smithi043=a;count:1 2024-05-15T15:29:49.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:49 smithi043 ceph-mon[24430]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:49.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:49 smithi043 ceph-mon[24430]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi043=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:49 smithi043 ceph-mon[24430]: Saving service alertmanager spec with placement smithi043=a;count:1 2024-05-15T15:29:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:49 smithi043 ceph-mon[29127]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:49 smithi043 ceph-mon[29127]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi043=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:49 smithi043 ceph-mon[29127]: Saving service alertmanager spec with placement smithi043=a;count:1 2024-05-15T15:29:51.729 INFO:teuthology.orchestra.run.smithi053.stdout:Scheduled grafana update... 2024-05-15T15:29:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:51 smithi053 ceph-mon[25205]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:51 smithi043 ceph-mon[24430]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:52.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:51 smithi043 ceph-mon[29127]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:52.631 DEBUG:teuthology.orchestra.run.smithi053:grafana.a> sudo journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@grafana.a.service 2024-05-15T15:29:52.633 INFO:tasks.cephadm:Setting up client nodes... 2024-05-15T15:29:52.636 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-15T15:29:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:52 smithi053 ceph-mon[25205]: from='client.24380 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi053=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:52 smithi053 ceph-mon[25205]: Saving service grafana spec with placement smithi053=a;count:1 2024-05-15T15:29:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:52 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:52.954 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:52 smithi043 ceph-mon[24430]: from='client.24380 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi053=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:52.955 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:52 smithi043 ceph-mon[24430]: Saving service grafana spec with placement smithi053=a;count:1 2024-05-15T15:29:52.955 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:52 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:52.955 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:52 smithi043 ceph-mon[29127]: from='client.24380 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi053=a", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:29:52.955 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:52 smithi043 ceph-mon[29127]: Saving service grafana spec with placement smithi053=a;count:1 2024-05-15T15:29:52.955 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:52 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:29:54.080 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:53 smithi043 ceph-mon[24430]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:54.081 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:53 smithi043 ceph-mon[29127]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:53 smithi053 ceph-mon[25205]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:54.748 INFO:teuthology.orchestra.run.smithi043.stdout:[client.0] 2024-05-15T15:29:54.748 INFO:teuthology.orchestra.run.smithi043.stdout: key = AQDy1ERmAkBbLBAARsW+ITWSRDmyoDnulOKNxQ== 2024-05-15T15:29:55.430 DEBUG:teuthology.orchestra.run.smithi043:> set -ex 2024-05-15T15:29:55.430 DEBUG:teuthology.orchestra.run.smithi043:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-05-15T15:29:55.430 DEBUG:teuthology.orchestra.run.smithi043:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-05-15T15:29:55.472 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-15T15:29:55.795 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:55 smithi053 ceph-mon[25205]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:55.796 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:55 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3350616606' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-15T15:29:55.796 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:55 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3350616606' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-15T15:29:56.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:55 smithi043 ceph-mon[24430]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:56.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:55 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3350616606' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-15T15:29:56.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:55 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3350616606' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-15T15:29:56.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:55 smithi043 ceph-mon[29127]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:56.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:55 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3350616606' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-15T15:29:56.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:55 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3350616606' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-15T15:29:58.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:57 smithi043 ceph-mon[24430]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:58.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:57 smithi043 ceph-mon[29127]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:58.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:57 smithi053 ceph-mon[25205]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:59.042 INFO:teuthology.orchestra.run.smithi053.stdout:[client.1] 2024-05-15T15:29:59.043 INFO:teuthology.orchestra.run.smithi053.stdout: key = AQD31ERmrhtEAhAAqPhqB6cNHp/fdBrk2EFYGQ== 2024-05-15T15:29:59.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:59 smithi053 ceph-mon[25205]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:29:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:59 smithi053 ceph-mon[25205]: from='client.? 172.21.15.53:0/1990873079' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-15T15:29:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:59 smithi053 ceph-mon[25205]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-15T15:29:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:29:59 smithi053 ceph-mon[25205]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-15T15:29:59.907 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-05-15T15:29:59.908 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-05-15T15:29:59.908 DEBUG:teuthology.orchestra.run.smithi053:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-05-15T15:29:59.951 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-05-15T15:29:59.951 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-05-15T15:29:59.951 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph mgr dump --format=json 2024-05-15T15:30:00.085 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:59 smithi043 ceph-mon[29127]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:00.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:59 smithi043 ceph-mon[29127]: from='client.? 172.21.15.53:0/1990873079' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-15T15:30:00.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:59 smithi043 ceph-mon[29127]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-15T15:30:00.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:29:59 smithi043 ceph-mon[29127]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-15T15:30:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:59 smithi043 ceph-mon[24430]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:59 smithi043 ceph-mon[24430]: from='client.? 172.21.15.53:0/1990873079' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-15T15:30:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:59 smithi043 ceph-mon[24430]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-15T15:30:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:29:59 smithi043 ceph-mon[24430]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-15T15:30:00.185 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:00.967 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:00 smithi043 ceph-mon[24430]: overall HEALTH_OK 2024-05-15T15:30:00.967 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:00 smithi043 ceph-mon[29127]: overall HEALTH_OK 2024-05-15T15:30:01.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:00 smithi053 ceph-mon[25205]: overall HEALTH_OK 2024-05-15T15:30:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:01 smithi043 ceph-mon[24430]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:02.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:01 smithi043 ceph-mon[29127]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:01 smithi053 ceph-mon[25205]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:02.137 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:30:02.750 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:02 smithi053 systemd[1]: Starting Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:30:02.798 INFO:teuthology.orchestra.run.smithi043.stdout:{"epoch":19,"active_gid":24325,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6800","nonce":3889110225},{"type":"v1","addr":"172.21.15.43:6801","nonce":3889110225}]},"active_addr":"172.21.15.43:6801/3889110225","active_change":"2024-05-15T15:29:44.199337+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24335,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.43:8443/","prometheus":"http://172.21.15.43:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.43:0","nonce":1094161927}]},{"addrvec":[{"type":"v2","addr":"172.21.15.43:0","nonce":4085531265}]},{"addrvec":[{"type":"v2","addr":"172.21.15.43:0","nonce":4292877123}]},{"addrvec":[{"type":"v2","addr":"172.21.15.43:0","nonce":699734202}]}]}} 2024-05-15T15:30:02.802 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-05-15T15:30:02.802 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-05-15T15:30:02.802 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd dump --format=json 2024-05-15T15:30:03.006 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:02 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/175250423' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-15T15:30:03.007 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:02 smithi053 podman[40179]: 2024-05-15 15:30:02.760836921 +0000 UTC m=+0.022071862 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-15T15:30:03.007 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:02 smithi053 podman[40179]: 2024-05-15 15:30:02.88291439 +0000 UTC m=+0.144149348 container create 0870489507f965174036d784810e2b458a087030ae5f90061549d49b5fd79fc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:30:03.027 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:03.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:02 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/175250423' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-15T15:30:03.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:02 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/175250423' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-15T15:30:03.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 podman[40179]: 2024-05-15 15:30:03.093494202 +0000 UTC m=+0.354729142 container init 0870489507f965174036d784810e2b458a087030ae5f90061549d49b5fd79fc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:30:03.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 podman[40179]: 2024-05-15 15:30:03.096702887 +0000 UTC m=+0.357937830 container start 0870489507f965174036d784810e2b458a087030ae5f90061549d49b5fd79fc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:30:03.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.121Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.121Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.121Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.121Z caller=main.go:518 level=info host_details="(Linux 5.14.0-446.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024 x86_64 smithi053 (none))" 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.121Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.121Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.122Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.123Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.124Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.126Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.126Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.977µs 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.126Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.126Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-05-15T15:30:03.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.126Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=33.912µs wal_replay_duration=212.327µs total_replay_duration=261.733µs 2024-05-15T15:30:03.383 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.127Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-15T15:30:03.383 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.127Z caller=main.go:947 level=info msg="TSDB started" 2024-05-15T15:30:03.383 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.127Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-15T15:30:03.383 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.143Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.131328ms db_storage=601ns remote_storage=1.142µs web_handler=268ns query_engine=530ns scrape=394.753µs scrape_sd=20.195µs notify=922ns notify_sd=1.378µs rules=15.504933ms 2024-05-15T15:30:03.383 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:30:03.143Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-15T15:30:03.383 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 bash[40179]: 0870489507f965174036d784810e2b458a087030ae5f90061549d49b5fd79fc5 2024-05-15T15:30:03.383 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:30:03 smithi053 systemd[1]: Started Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:30:04.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:03 smithi043 ceph-mon[24430]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:04.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:03 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:04.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:03 smithi043 ceph-mon[29127]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:04.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:03 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:03 smithi053 ceph-mon[25205]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:04.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:03 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:04.947 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:30:04.947 INFO:teuthology.orchestra.run.smithi043.stdout:{"epoch":60,"fsid":"22d82864-12cf-11ef-bc98-c7b262605968","created":"2024-05-15T15:24:28.633262+0000","modified":"2024-05-15T15:29:44.199008+0000","last_up_change":"2024-05-15T15:29:28.553820+0000","last_in_change":"2024-05-15T15:29:11.147309+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-15T15:27:32.341598+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7ac451f1-36bd-4ff8-8b90-92a83e712159","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6802","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6803","nonce":2576871983}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6804","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6805","nonce":2576871983}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6808","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6809","nonce":2576871983}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6806","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6807","nonce":2576871983}]},"public_addr":"172.21.15.43:6803/2576871983","cluster_addr":"172.21.15.43:6805/2576871983","heartbeat_back_addr":"172.21.15.43:6809/2576871983","heartbeat_front_addr":"172.21.15.43:6807/2576871983","state":["exists","up"]},{"osd":1,"uuid":"c35bd4a6-ea35-419f-a5cc-fc97f9eb509d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6810","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6811","nonce":3463535224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6812","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6813","nonce":3463535224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6816","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6817","nonce":3463535224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6814","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6815","nonce":3463535224}]},"public_addr":"172.21.15.43:6811/3463535224","cluster_addr":"172.21.15.43:6813/3463535224","heartbeat_back_addr":"172.21.15.43:6817/3463535224","heartbeat_front_addr":"172.21.15.43:6815/3463535224","state":["exists","up"]},{"osd":2,"uuid":"7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6818","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6819","nonce":50306098}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6820","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6821","nonce":50306098}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6824","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6825","nonce":50306098}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6822","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6823","nonce":50306098}]},"public_addr":"172.21.15.43:6819/50306098","cluster_addr":"172.21.15.43:6821/50306098","heartbeat_back_addr":"172.21.15.43:6825/50306098","heartbeat_front_addr":"172.21.15.43:6823/50306098","state":["exists","up"]},{"osd":3,"uuid":"f79328f3-261d-4a48-8a97-952a3bf9b97e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6826","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6827","nonce":3944497486}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6828","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6829","nonce":3944497486}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6832","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6833","nonce":3944497486}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6830","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6831","nonce":3944497486}]},"public_addr":"172.21.15.43:6827/3944497486","cluster_addr":"172.21.15.43:6829/3944497486","heartbeat_back_addr":"172.21.15.43:6833/3944497486","heartbeat_front_addr":"172.21.15.43:6831/3944497486","state":["exists","up"]},{"osd":4,"uuid":"50ed214e-19e1-4bab-95ac-d636276f8925","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6800","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6801","nonce":315819048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6802","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6803","nonce":315819048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6806","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6807","nonce":315819048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6804","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6805","nonce":315819048}]},"public_addr":"172.21.15.53:6801/315819048","cluster_addr":"172.21.15.53:6803/315819048","heartbeat_back_addr":"172.21.15.53:6807/315819048","heartbeat_front_addr":"172.21.15.53:6805/315819048","state":["exists","up"]},{"osd":5,"uuid":"4ab4ad5a-e5eb-46f2-a41f-82458b9b84af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6808","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6809","nonce":2835156943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6810","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6811","nonce":2835156943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6814","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6815","nonce":2835156943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6812","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6813","nonce":2835156943}]},"public_addr":"172.21.15.53:6809/2835156943","cluster_addr":"172.21.15.53:6811/2835156943","heartbeat_back_addr":"172.21.15.53:6815/2835156943","heartbeat_front_addr":"172.21.15.53:6813/2835156943","state":["exists","up"]},{"osd":6,"uuid":"11502520-c379-4b3d-a9d7-db332bd28cb6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6816","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6817","nonce":2855453996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6818","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6819","nonce":2855453996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6822","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6823","nonce":2855453996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6820","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6821","nonce":2855453996}]},"public_addr":"172.21.15.53:6817/2855453996","cluster_addr":"172.21.15.53:6819/2855453996","heartbeat_back_addr":"172.21.15.53:6823/2855453996","heartbeat_front_addr":"172.21.15.53:6821/2855453996","state":["exists","up"]},{"osd":7,"uuid":"12f9ad71-027e-4836-8508-662d813d3f0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6824","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6825","nonce":3221065522}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6826","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6827","nonce":3221065522}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6830","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6831","nonce":3221065522}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6828","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6829","nonce":3221065522}]},"public_addr":"172.21.15.53:6825/3221065522","cluster_addr":"172.21.15.53:6827/3221065522","heartbeat_back_addr":"172.21.15.53:6831/3221065522","heartbeat_front_addr":"172.21.15.53:6829/3221065522","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:26:41.970916+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:27:05.203106+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:27:28.889033+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:27:54.138937+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:28:16.062057+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:28:39.548674+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:29:02.818322+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:29:26.663626+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.43:6801/3688153742":"2024-05-16T15:29:44.198974+0000","172.21.15.43:6800/3688153742":"2024-05-16T15:29:44.198974+0000","172.21.15.43:0/2314145657":"2024-05-16T15:29:44.198974+0000","172.21.15.43:0/4208721262":"2024-05-16T15:29:44.198974+0000","172.21.15.43:0/3975126779":"2024-05-16T15:25:23.773919+0000","172.21.15.43:6800/970314706":"2024-05-16T15:25:23.773919+0000","172.21.15.43:0/1710373736":"2024-05-16T15:25:23.773919+0000","172.21.15.43:0/3368735601":"2024-05-16T15:29:44.198974+0000","172.21.15.43:0/573610528":"2024-05-16T15:25:23.773919+0000","172.21.15.43:6800/2411931462":"2024-05-16T15:25:01.016490+0000","172.21.15.43:6801/2411931462":"2024-05-16T15:25:01.016490+0000","172.21.15.43:0/653624395":"2024-05-16T15:25:01.016490+0000","172.21.15.43:0/1103755388":"2024-05-16T15:25:01.016490+0000","172.21.15.43:0/663705201":"2024-05-16T15:29:44.198974+0000","172.21.15.43:6801/970314706":"2024-05-16T15:25:23.773919+0000","172.21.15.43:0/3154897789":"2024-05-16T15:25:01.016490+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-15T15:30:05.031 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:04 smithi053 ceph-mon[25205]: Deploying daemon node-exporter.a on smithi043 2024-05-15T15:30:05.031 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:04 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:05.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:04 smithi043 ceph-mon[24430]: Deploying daemon node-exporter.a on smithi043 2024-05-15T15:30:05.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:04 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:05.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:04 smithi043 ceph-mon[29127]: Deploying daemon node-exporter.a on smithi043 2024-05-15T15:30:05.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:04 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:05.682 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-05-15T15:30:05.682 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd dump --format=json 2024-05-15T15:30:05.912 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:05 smithi043 ceph-mon[29127]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:05.912 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:05 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1135320933' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-15T15:30:05.912 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:05 smithi043 ceph-mon[24430]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:05.912 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:05 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1135320933' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-15T15:30:05.945 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:06.056 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:05 smithi053 ceph-mon[25205]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:06.056 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:05 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1135320933' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-15T15:30:06.586 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:06 smithi043 bash[43993]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-05-15T15:30:08.049 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:07 smithi043 ceph-mon[24430]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:08.050 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:07 smithi043 ceph-mon[29127]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:07 smithi053 ceph-mon[25205]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:08.303 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:08 smithi043 bash[43993]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-05-15T15:30:08.621 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:08 smithi043 bash[43993]: Writing manifest to image destination 2024-05-15T15:30:08.775 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:30:08.776 INFO:teuthology.orchestra.run.smithi043.stdout:{"epoch":60,"fsid":"22d82864-12cf-11ef-bc98-c7b262605968","created":"2024-05-15T15:24:28.633262+0000","modified":"2024-05-15T15:29:44.199008+0000","last_up_change":"2024-05-15T15:29:28.553820+0000","last_in_change":"2024-05-15T15:29:11.147309+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-15T15:27:32.341598+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7ac451f1-36bd-4ff8-8b90-92a83e712159","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6802","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6803","nonce":2576871983}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6804","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6805","nonce":2576871983}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6808","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6809","nonce":2576871983}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6806","nonce":2576871983},{"type":"v1","addr":"172.21.15.43:6807","nonce":2576871983}]},"public_addr":"172.21.15.43:6803/2576871983","cluster_addr":"172.21.15.43:6805/2576871983","heartbeat_back_addr":"172.21.15.43:6809/2576871983","heartbeat_front_addr":"172.21.15.43:6807/2576871983","state":["exists","up"]},{"osd":1,"uuid":"c35bd4a6-ea35-419f-a5cc-fc97f9eb509d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6810","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6811","nonce":3463535224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6812","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6813","nonce":3463535224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6816","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6817","nonce":3463535224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6814","nonce":3463535224},{"type":"v1","addr":"172.21.15.43:6815","nonce":3463535224}]},"public_addr":"172.21.15.43:6811/3463535224","cluster_addr":"172.21.15.43:6813/3463535224","heartbeat_back_addr":"172.21.15.43:6817/3463535224","heartbeat_front_addr":"172.21.15.43:6815/3463535224","state":["exists","up"]},{"osd":2,"uuid":"7d5d8c6f-1506-4d81-8b8c-b0a18b1f8243","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6818","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6819","nonce":50306098}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6820","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6821","nonce":50306098}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6824","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6825","nonce":50306098}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6822","nonce":50306098},{"type":"v1","addr":"172.21.15.43:6823","nonce":50306098}]},"public_addr":"172.21.15.43:6819/50306098","cluster_addr":"172.21.15.43:6821/50306098","heartbeat_back_addr":"172.21.15.43:6825/50306098","heartbeat_front_addr":"172.21.15.43:6823/50306098","state":["exists","up"]},{"osd":3,"uuid":"f79328f3-261d-4a48-8a97-952a3bf9b97e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6826","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6827","nonce":3944497486}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6828","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6829","nonce":3944497486}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6832","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6833","nonce":3944497486}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.43:6830","nonce":3944497486},{"type":"v1","addr":"172.21.15.43:6831","nonce":3944497486}]},"public_addr":"172.21.15.43:6827/3944497486","cluster_addr":"172.21.15.43:6829/3944497486","heartbeat_back_addr":"172.21.15.43:6833/3944497486","heartbeat_front_addr":"172.21.15.43:6831/3944497486","state":["exists","up"]},{"osd":4,"uuid":"50ed214e-19e1-4bab-95ac-d636276f8925","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6800","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6801","nonce":315819048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6802","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6803","nonce":315819048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6806","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6807","nonce":315819048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6804","nonce":315819048},{"type":"v1","addr":"172.21.15.53:6805","nonce":315819048}]},"public_addr":"172.21.15.53:6801/315819048","cluster_addr":"172.21.15.53:6803/315819048","heartbeat_back_addr":"172.21.15.53:6807/315819048","heartbeat_front_addr":"172.21.15.53:6805/315819048","state":["exists","up"]},{"osd":5,"uuid":"4ab4ad5a-e5eb-46f2-a41f-82458b9b84af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6808","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6809","nonce":2835156943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6810","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6811","nonce":2835156943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6814","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6815","nonce":2835156943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6812","nonce":2835156943},{"type":"v1","addr":"172.21.15.53:6813","nonce":2835156943}]},"public_addr":"172.21.15.53:6809/2835156943","cluster_addr":"172.21.15.53:6811/2835156943","heartbeat_back_addr":"172.21.15.53:6815/2835156943","heartbeat_front_addr":"172.21.15.53:6813/2835156943","state":["exists","up"]},{"osd":6,"uuid":"11502520-c379-4b3d-a9d7-db332bd28cb6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6816","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6817","nonce":2855453996}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6818","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6819","nonce":2855453996}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6822","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6823","nonce":2855453996}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6820","nonce":2855453996},{"type":"v1","addr":"172.21.15.53:6821","nonce":2855453996}]},"public_addr":"172.21.15.53:6817/2855453996","cluster_addr":"172.21.15.53:6819/2855453996","heartbeat_back_addr":"172.21.15.53:6823/2855453996","heartbeat_front_addr":"172.21.15.53:6821/2855453996","state":["exists","up"]},{"osd":7,"uuid":"12f9ad71-027e-4836-8508-662d813d3f0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6824","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6825","nonce":3221065522}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6826","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6827","nonce":3221065522}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6830","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6831","nonce":3221065522}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6828","nonce":3221065522},{"type":"v1","addr":"172.21.15.53:6829","nonce":3221065522}]},"public_addr":"172.21.15.53:6825/3221065522","cluster_addr":"172.21.15.53:6827/3221065522","heartbeat_back_addr":"172.21.15.53:6831/3221065522","heartbeat_front_addr":"172.21.15.53:6829/3221065522","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:26:41.970916+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:27:05.203106+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:27:28.889033+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:27:54.138937+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:28:16.062057+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:28:39.548674+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:29:02.818322+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-15T15:29:26.663626+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.43:6801/3688153742":"2024-05-16T15:29:44.198974+0000","172.21.15.43:6800/3688153742":"2024-05-16T15:29:44.198974+0000","172.21.15.43:0/2314145657":"2024-05-16T15:29:44.198974+0000","172.21.15.43:0/4208721262":"2024-05-16T15:29:44.198974+0000","172.21.15.43:0/3975126779":"2024-05-16T15:25:23.773919+0000","172.21.15.43:6800/970314706":"2024-05-16T15:25:23.773919+0000","172.21.15.43:0/1710373736":"2024-05-16T15:25:23.773919+0000","172.21.15.43:0/3368735601":"2024-05-16T15:29:44.198974+0000","172.21.15.43:0/573610528":"2024-05-16T15:25:23.773919+0000","172.21.15.43:6800/2411931462":"2024-05-16T15:25:01.016490+0000","172.21.15.43:6801/2411931462":"2024-05-16T15:25:01.016490+0000","172.21.15.43:0/653624395":"2024-05-16T15:25:01.016490+0000","172.21.15.43:0/1103755388":"2024-05-16T15:25:01.016490+0000","172.21.15.43:0/663705201":"2024-05-16T15:29:44.198974+0000","172.21.15.43:6801/970314706":"2024-05-16T15:25:23.773919+0000","172.21.15.43:0/3154897789":"2024-05-16T15:25:01.016490+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-15T15:30:08.917 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:08 smithi043 podman[43993]: 2024-05-15 15:30:08.707585231 +0000 UTC m=+2.919020441 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-15T15:30:08.917 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:08 smithi043 podman[43993]: 2024-05-15 15:30:08.781003531 +0000 UTC m=+2.992438743 container create 8cfbd4656c29d3dea493c8e3a39ca725039c920d6246582e3d5836872c449b35 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:30:09.570 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-05-15T15:30:09.571 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-05-15T15:30:09.571 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-05-15T15:30:09.571 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-05-15T15:30:09.572 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-05-15T15:30:09.572 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-05-15T15:30:09.573 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-05-15T15:30:09.573 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-05-15T15:30:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:09 smithi043 ceph-mon[29127]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:09.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:09 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3012399354' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-15T15:30:09.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 podman[43993]: 2024-05-15 15:30:09.662808045 +0000 UTC m=+3.874243275 container init 8cfbd4656c29d3dea493c8e3a39ca725039c920d6246582e3d5836872c449b35 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:30:09.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 podman[43993]: 2024-05-15 15:30:09.671160108 +0000 UTC m=+3.882595325 container start 8cfbd4656c29d3dea493c8e3a39ca725039c920d6246582e3d5836872c449b35 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:30:09.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.713Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-15T15:30:09.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.713Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-15T15:30:09.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.715Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-15T15:30:09.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.715Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-15T15:30:09.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.716Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-15T15:30:09.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.716Z caller=node_exporter.go:115 level=info collector=arp 2024-05-15T15:30:09.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.716Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-15T15:30:09.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.716Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-15T15:30:09.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.716Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-15T15:30:09.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.716Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-15T15:30:09.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.717Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-15T15:30:09.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.717Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-15T15:30:09.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.717Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.717Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.717Z caller=node_exporter.go:115 level=info collector=edac 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.717Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.717Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.717Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.717Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.717Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.718Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.718Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.718Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.718Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-15T15:30:09.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.718Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.718Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.718Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.718Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.719Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.719Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.719Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.719Z caller=node_exporter.go:115 level=info collector=os 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.719Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.719Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.719Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.719Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-15T15:30:09.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.720Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-15T15:30:09.841 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.720Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-15T15:30:09.841 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.720Z caller=node_exporter.go:115 level=info collector=stat 2024-05-15T15:30:09.841 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.720Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-15T15:30:09.841 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.720Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-15T15:30:09.841 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.720Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-15T15:30:09.841 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.720Z caller=node_exporter.go:115 level=info collector=time 2024-05-15T15:30:09.841 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.721Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-15T15:30:09.842 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.721Z caller=node_exporter.go:115 level=info collector=uname 2024-05-15T15:30:09.842 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.721Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-15T15:30:09.842 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.721Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-15T15:30:09.842 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.721Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-15T15:30:09.842 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.721Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-15T15:30:09.842 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[44164]: ts=2024-05-15T15:30:09.722Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-15T15:30:09.842 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 bash[43993]: 8cfbd4656c29d3dea493c8e3a39ca725039c920d6246582e3d5836872c449b35 2024-05-15T15:30:09.842 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:30:09 smithi043 systemd[1]: Started Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:30:09.842 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-mon[24430]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:09.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:09 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3012399354' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-15T15:30:10.001 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:10.023 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:09 smithi053 ceph-mon[25205]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:10.023 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:09 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3012399354' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-15T15:30:10.031 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:10.032 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:10.035 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:10.056 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:10.500 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:10.845 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:10 smithi053 systemd[1]: Starting Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:30:10.930 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:10 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:10 smithi053 ceph-mon[25205]: Deploying daemon node-exporter.b on smithi053 2024-05-15T15:30:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:10 smithi053 ceph-mon[25205]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:11.130 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:10 smithi053 bash[40422]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-05-15T15:30:11.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:10 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:11.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:10 smithi043 ceph-mon[24430]: Deploying daemon node-exporter.b on smithi053 2024-05-15T15:30:11.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:10 smithi043 ceph-mon[24430]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:11.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:10 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:11.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:10 smithi043 ceph-mon[29127]: Deploying daemon node-exporter.b on smithi053 2024-05-15T15:30:11.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:10 smithi043 ceph-mon[29127]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:11.880 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:11 smithi053 bash[40422]: Getting image source signatures 2024-05-15T15:30:11.880 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:11 smithi053 bash[40422]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-05-15T15:30:11.880 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:11 smithi053 bash[40422]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-05-15T15:30:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:11 smithi053 bash[40422]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-05-15T15:30:12.766 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:30:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:30:12] "GET /metrics HTTP/1.1" 200 191508 "" "Prometheus/2.33.4" 2024-05-15T15:30:13.530 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:13 smithi043 ceph-mon[24430]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:13.530 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:13 smithi043 ceph-mon[29127]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:13 smithi053 ceph-mon[25205]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:14.130 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:13 smithi053 bash[40422]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-05-15T15:30:14.130 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:13 smithi053 bash[40422]: Writing manifest to image destination 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 podman[40422]: 2024-05-15 15:30:14.252918172 +0000 UTC m=+3.505962526 container create c6fac2ae2513753713036c847f9891b539cad8ab0c52331cf5d0e676ef46c469 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 podman[40422]: 2024-05-15 15:30:14.17995721 +0000 UTC m=+3.433001573 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 podman[40422]: 2024-05-15 15:30:14.446438167 +0000 UTC m=+3.699482524 container init c6fac2ae2513753713036c847f9891b539cad8ab0c52331cf5d0e676ef46c469 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 podman[40422]: 2024-05-15 15:30:14.449771062 +0000 UTC m=+3.702815427 container start c6fac2ae2513753713036c847f9891b539cad8ab0c52331cf5d0e676ef46c469 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.454Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.455Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=arp 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-15T15:30:14.532 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=edac 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-15T15:30:14.533 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=os 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-15T15:30:14.534 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=stat 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=time 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=uname 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-15T15:30:14.535 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-15T15:30:14.536 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.456Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-15T15:30:14.536 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[40471]: ts=2024-05-15T15:30:14.457Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-15T15:30:14.536 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 bash[40422]: c6fac2ae2513753713036c847f9891b539cad8ab0c52331cf5d0e676ef46c469 2024-05-15T15:30:14.538 INFO:teuthology.orchestra.run.smithi043.stdout:77309411363 2024-05-15T15:30:14.538 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-05-15T15:30:14.880 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:30:14 smithi053 systemd[1]: Started Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:30:15.603 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:15.612 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:15 smithi043 ceph-mon[24430]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:15.612 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:15 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:15.612 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:15 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:30:15.612 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:15 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:30:15.612 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:15 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:30:15.612 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:15 smithi043 ceph-mon[29127]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:15.613 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:15 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:15.613 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:15 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:30:15.613 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:15 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:30:15.613 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:15 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:30:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:15 smithi053 ceph-mon[25205]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:15 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:15 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:30:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:15 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:30:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:15 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:30:17.042 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:30:16 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:30:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:30:17.172 INFO:teuthology.orchestra.run.smithi043.stdout:133143986202 2024-05-15T15:30:17.172 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-05-15T15:30:17.865 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:17 smithi043 ceph-mon[24430]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:17.865 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:17 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:17.866 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:17 smithi043 ceph-mon[29127]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:17.866 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:17 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:17 smithi053 ceph-mon[25205]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:17 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:19.744 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:19 smithi043 ceph-mon[24430]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:19.744 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:19 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:19.744 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:19 smithi043 ceph-mon[29127]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:19.744 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:19 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:19 smithi053 ceph-mon[25205]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:19 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:20.104 INFO:teuthology.orchestra.run.smithi043.stdout:236223201291 2024-05-15T15:30:20.104 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-05-15T15:30:21.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:21 smithi043 ceph-mon[24430]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:21 smithi043 ceph-mon[29127]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:22.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:21 smithi053 ceph-mon[25205]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:22.446 INFO:teuthology.orchestra.run.smithi043.stdout:167503724566 2024-05-15T15:30:22.447 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-05-15T15:30:22.580 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:22.581 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:22.582 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:22.671 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:30:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:30:22] "GET /metrics HTTP/1.1" 200 191483 "" "Prometheus/2.33.4" 2024-05-15T15:30:22.977 INFO:teuthology.orchestra.run.smithi043.stdout:111669149726 2024-05-15T15:30:22.977 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-05-15T15:30:23.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:23 smithi043 ceph-mon[24430]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:23 smithi043 ceph-mon[29127]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:24.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:23 smithi053 ceph-mon[25205]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:26.053 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:25 smithi043 ceph-mon[29127]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:26.053 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:25 smithi043 ceph-mon[24430]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:26.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:25 smithi053 ceph-mon[25205]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:27.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:30:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:30:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:30:27.465 INFO:teuthology.orchestra.run.smithi043.stdout:236223201293 2024-05-15T15:30:27.683 INFO:teuthology.orchestra.run.smithi043.stdout:34359738413 2024-05-15T15:30:27.684 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-05-15T15:30:27.835 INFO:teuthology.orchestra.run.smithi043.stdout:201863462931 2024-05-15T15:30:27.835 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-05-15T15:30:27.897 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:27.898 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:27 smithi043 ceph-mon[29127]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:27 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/741291613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-15T15:30:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:27 smithi043 ceph-mon[24430]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:27 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/741291613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-15T15:30:28.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:27 smithi053 ceph-mon[25205]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:28.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:27 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/741291613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-15T15:30:29.135 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:29.679 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201291 got 236223201293 for osd.7 2024-05-15T15:30:29.679 DEBUG:teuthology.parallel:result is None 2024-05-15T15:30:29.950 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:29 smithi043 ceph-mon[24430]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:29.950 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:29 smithi043 ceph-mon[29127]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:29 smithi053 ceph-mon[25205]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:32.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:31 smithi043 ceph-mon[24430]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:32.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:31 smithi043 ceph-mon[29127]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:31 smithi053 ceph-mon[25205]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:32.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:30:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:30:32] "GET /metrics HTTP/1.1" 200 191464 "" "Prometheus/2.33.4" 2024-05-15T15:30:33.446 INFO:teuthology.orchestra.run.smithi043.stdout:133143986205 2024-05-15T15:30:33.960 INFO:teuthology.orchestra.run.smithi043.stdout:77309411367 2024-05-15T15:30:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:33 smithi043 ceph-mon[24430]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:33 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1389324676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-15T15:30:34.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:33 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3649032508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-15T15:30:34.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:33 smithi043 ceph-mon[29127]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:34.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:33 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1389324676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-15T15:30:34.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:33 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3649032508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-15T15:30:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:33 smithi053 ceph-mon[25205]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:33 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1389324676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-15T15:30:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:33 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3649032508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-15T15:30:34.647 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411363 got 77309411367 for osd.2 2024-05-15T15:30:34.647 DEBUG:teuthology.parallel:result is None 2024-05-15T15:30:35.111 INFO:teuthology.orchestra.run.smithi043.stdout:55834574890 2024-05-15T15:30:35.112 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-05-15T15:30:36.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:35 smithi043 ceph-mon[24430]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:36.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:35 smithi043 ceph-mon[29127]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:36.129 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:35 smithi053 ceph-mon[25205]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:36.452 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:37.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:36 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3305902281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-15T15:30:37.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:36 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3305902281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-15T15:30:37.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:36 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3305902281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-15T15:30:37.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:30:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:30:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:30:37.746 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986202 got 133143986205 for osd.4 2024-05-15T15:30:37.746 DEBUG:teuthology.parallel:result is None 2024-05-15T15:30:37.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:37 smithi043 ceph-mon[24430]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:37 smithi043 ceph-mon[29127]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:37.837 INFO:teuthology.orchestra.run.smithi043.stdout:201863462932 2024-05-15T15:30:38.129 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:37 smithi053 ceph-mon[25205]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:38.454 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462931 got 201863462932 for osd.6 2024-05-15T15:30:38.454 DEBUG:teuthology.parallel:result is None 2024-05-15T15:30:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:39 smithi043 ceph-mon[24430]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:39 smithi043 ceph-mon[29127]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:40.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:39 smithi053 ceph-mon[25205]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:40.506 INFO:teuthology.orchestra.run.smithi043.stdout:34359738416 2024-05-15T15:30:40.895 INFO:teuthology.orchestra.run.smithi043.stdout:167503724569 2024-05-15T15:30:40.895 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:40 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2427453438' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-15T15:30:40.895 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:40 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2427453438' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-15T15:30:41.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:40 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2427453438' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-15T15:30:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:41 smithi043 ceph-mon[24430]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:41 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1043607923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-15T15:30:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:41 smithi043 ceph-mon[29127]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:41 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1043607923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-15T15:30:41.992 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724566 got 167503724569 for osd.5 2024-05-15T15:30:41.993 DEBUG:teuthology.parallel:result is None 2024-05-15T15:30:42.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:41 smithi053 ceph-mon[25205]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:42.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:41 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1043607923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-15T15:30:42.648 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738413 got 34359738416 for osd.0 2024-05-15T15:30:42.648 DEBUG:teuthology.parallel:result is None 2024-05-15T15:30:42.917 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:30:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:30:42] "GET /metrics HTTP/1.1" 200 191464 "" "Prometheus/2.33.4" 2024-05-15T15:30:42.921 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:43.098 INFO:teuthology.orchestra.run.smithi043.stdout:111669149729 2024-05-15T15:30:43.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:43 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:43.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:43 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:43.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:43 smithi043 ceph-mon[24430]: Deploying daemon alertmanager.a on smithi043 2024-05-15T15:30:43.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:43 smithi043 ceph-mon[24430]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:43.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:43 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/293988368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-15T15:30:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:43 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:43 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:43 smithi043 ceph-mon[29127]: Deploying daemon alertmanager.a on smithi043 2024-05-15T15:30:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:43 smithi043 ceph-mon[29127]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:43 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/293988368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-15T15:30:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:43 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:43 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:43 smithi053 ceph-mon[25205]: Deploying daemon alertmanager.a on smithi043 2024-05-15T15:30:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:43 smithi053 ceph-mon[25205]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:43 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/293988368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-15T15:30:43.746 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149726 got 111669149729 for osd.3 2024-05-15T15:30:43.746 DEBUG:teuthology.parallel:result is None 2024-05-15T15:30:45.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:45 smithi043 ceph-mon[29127]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:45.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:30:45.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:45 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:30:45.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:45 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:30:45.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:45 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:30:45.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:45 smithi043 ceph-mon[24430]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:45.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:30:45.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:45 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:30:45.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:45 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:30:45.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:45 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:30:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:45 smithi053 ceph-mon[25205]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:30:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:45 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:30:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:45 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:30:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:45 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:30:45.760 INFO:teuthology.orchestra.run.smithi043.stdout:55834574892 2024-05-15T15:30:46.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:46 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2496678797' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-15T15:30:46.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:46 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2496678797' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-15T15:30:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:46 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2496678797' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-15T15:30:47.111 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574892 for osd.1 2024-05-15T15:30:47.111 DEBUG:teuthology.parallel:result is None 2024-05-15T15:30:47.111 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-05-15T15:30:47.112 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph pg dump --format=json 2024-05-15T15:30:47.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:30:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:30:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:30:47.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:47 smithi043 ceph-mon[24430]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:47.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:47 smithi043 ceph-mon[29127]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:47.628 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:47 smithi053 ceph-mon[25205]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:47.952 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:49.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:49 smithi043 ceph-mon[24430]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:49.550 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:49 smithi043 ceph-mon[29127]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:49 smithi053 ceph-mon[25205]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:51.235 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:51 smithi043 systemd[1]: Starting Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:30:51.236 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:30:51.237 INFO:teuthology.orchestra.run.smithi043.stderr:dumped all 2024-05-15T15:30:51.567 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:51 smithi043 podman[47156]: 2024-05-15 15:30:51.203950737 +0000 UTC m=+0.017855072 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-05-15T15:30:51.567 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:51 smithi043 podman[47156]: 2024-05-15 15:30:51.325010139 +0000 UTC m=+0.138914466 volume create c9d288a5cf730292ba58970e8581c20662971abd54b9c9e334d1416b8b36c890 2024-05-15T15:30:51.567 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:51 smithi043 podman[47156]: 2024-05-15 15:30:51.40853601 +0000 UTC m=+0.222440329 container create 5851db4fb3e109f623095ed98cae0656f073c10c718f26f55995bfbd70e6c104 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:30:51.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:51 smithi043 ceph-mon[24430]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:51 smithi043 ceph-mon[29127]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:52.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:51 smithi053 ceph-mon[25205]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:52.237 INFO:teuthology.orchestra.run.smithi043.stdout:{"pg_ready":true,"pg_map":{"version":36,"stamp":"2024-05-15T15:30:50.271948+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48972,"kb_used_data":6156,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912244,"statfs":{"total":767960285184,"available":767910137856,"internally_reserved":0,"allocated":6303744,"data_stored":3511261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002241"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-05-15T15:29:44.339514+0000","last_change":"2024-05-15T15:29:32.577058+0000","last_active":"2024-05-15T15:29:44.339514+0000","last_peered":"2024-05-15T15:29:44.339514+0000","last_clean":"2024-05-15T15:29:44.339514+0000","last_became_active":"2024-05-15T15:29:32.576685+0000","last_became_peered":"2024-05-15T15:29:32.576685+0000","last_unstale":"2024-05-15T15:29:44.339514+0000","last_undegraded":"2024-05-15T15:29:44.339514+0000","last_fullsized":"2024-05-15T15:29:44.339514+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-15T15:27:32.692157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-15T15:27:32.692157+0000","last_clean_scrub_stamp":"2024-05-15T15:27:32.692157+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-16T16:20:34.823822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201297,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.44},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.53500000000000003}]},{"osd":1,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.621}]},{"osd":2,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.64700000000000002}]},{"osd":3,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.68600000000000005}]},{"osd":4,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.58299999999999996}]},{"osd":5,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.70199999999999996}]},{"osd":6,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.752},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.76300000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462935,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":769050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.47799999999999998}]},{"osd":1,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.52500000000000002}]},{"osd":2,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.38100000000000001}]},{"osd":3,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.497}]},{"osd":4,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.46600000000000003}]},{"osd":5,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.56699999999999995}]},{"osd":7,"last update":"Wed May 15 15:30:32 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.58999999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:16 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.35999999999999999,"15min":0.34100000000000003},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.38700000000000001,"15min":0.35899999999999999},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.53800000000000003}]},{"osd":2,"last update":"Wed May 15 15:30:38 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.44700000000000001,"15min":0.439},"min":{"1min":0.314,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40300000000000002},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.51200000000000001}]},{"osd":3,"last update":"Wed May 15 15:30:01 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.5,"15min":0.497},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.45500000000000002}]},{"osd":4,"last update":"Wed May 15 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.64400000000000002,"15min":0.65200000000000002},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.748,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79200000000000004,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.64700000000000002}]},{"osd":5,"last update":"Wed May 15 15:29:42 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.63400000000000001}]},{"osd":6,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.71999999999999997}]},{"osd":7,"last update":"Wed May 15 15:30:30 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.82799999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed May 15 15:30:10 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.36499999999999999,"15min":0.34699999999999998},"min":{"1min":0.30599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.59299999999999997,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.40200000000000002,"15min":0.38800000000000001},"min":{"1min":0.28299999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.34499999999999997}]},{"osd":2,"last update":"Wed May 15 15:30:35 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.39900000000000002,"15min":0.39100000000000001},"min":{"1min":0.30199999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.59299999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.40500000000000003,"15min":0.40699999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54500000000000004,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.27100000000000002}]},{"osd":3,"last update":"Wed May 15 15:30:02 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46500000000000002,"15min":0.46600000000000003},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.47999999999999998,"15min":0.48499999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.625,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.439}]},{"osd":4,"last update":"Wed May 15 15:30:22 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.59999999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76100000000000001,"5min":1.018,"15min":1.018},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.59899999999999998,"15min":0.60499999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74299999999999999,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.59299999999999997}]},{"osd":5,"last update":"Wed May 15 15:30:46 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.59299999999999997,"15min":0.58999999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.745,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.624,"15min":0.626},"min":{"1min":0.51300000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.81999999999999995,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.72099999999999997}]},{"osd":6,"last update":"Wed May 15 15:30:09 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.754}]},{"osd":7,"last update":"Wed May 15 15:30:29 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.66300000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:39 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56200000000000006,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.39200000000000002,"15min":0.40100000000000002},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.48499999999999999,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.33100000000000002}]},{"osd":1,"last update":"Wed May 15 15:30:39 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.40500000000000003,"15min":0.41599999999999998},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.61399999999999999,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.436,"15min":0.443},"min":{"1min":0.28299999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.50800000000000001,"5min":0.75,"15min":0.75},"last":0.29799999999999999}]},{"osd":3,"last update":"Wed May 15 15:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.48199999999999998,"15min":0.48599999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.52200000000000002,"15min":0.52800000000000002},"min":{"1min":0.39300000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.66500000000000004,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.34499999999999997}]},{"osd":4,"last update":"Wed May 15 15:30:28 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.80300000000000005,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.46000000000000002}]},{"osd":5,"last update":"Wed May 15 15:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.59299999999999997,"15min":0.58699999999999997},"min":{"1min":0.47099999999999997,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.57499999999999996,"15min":0.57599999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.45200000000000001}]},{"osd":6,"last update":"Wed May 15 15:30:06 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.046,"5min":1.046,"15min":1.046},"last":0.49299999999999999}]},{"osd":7,"last update":"Wed May 15 15:30:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.55900000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149732,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:29:58 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.40899999999999997,"15min":0.41199999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.60299999999999998,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.443,"15min":0.44700000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.61199999999999999,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.37}]},{"osd":1,"last update":"Wed May 15 15:29:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.442,"15min":0.442},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.497},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.433,"15min":0.435},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.58899999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.51300000000000001}]},{"osd":2,"last update":"Wed May 15 15:29:58 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.47399999999999998,"15min":0.47999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.59199999999999997,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.46600000000000003,"15min":0.46999999999999997},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.55000000000000004,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.60799999999999998}]},{"osd":4,"last update":"Wed May 15 15:30:21 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.623,"15min":0.61799999999999999},"min":{"1min":0.46000000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.63300000000000001,"15min":0.629},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.70899999999999996}]},{"osd":5,"last update":"Wed May 15 15:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.61699999999999999,"15min":0.60199999999999998},"min":{"1min":0.47999999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.64000000000000001,"15min":0.63100000000000001},"min":{"1min":0.434,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.624}]},{"osd":6,"last update":"Wed May 15 15:30:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.58399999999999996}]},{"osd":7,"last update":"Wed May 15 15:30:29 2024","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.57199999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986208,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.58399999999999996,"15min":0.59599999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69899999999999995,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.44},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.53100000000000003,"15min":0.53400000000000003},"min":{"1min":0.39200000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.66400000000000003}]},{"osd":1,"last update":"Wed May 15 15:30:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.65900000000000003,"5min":0.746,"15min":0.746},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.56799999999999995,"15min":0.57099999999999995},"min":{"1min":0.36399999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.75900000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.48799999999999999}]},{"osd":2,"last update":"Wed May 15 15:30:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.39200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.70899999999999996,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.621},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.57799999999999996,"15min":0.58399999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66900000000000004,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.58899999999999997}]},{"osd":3,"last update":"Wed May 15 15:30:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.57699999999999996,"15min":0.58299999999999996},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68500000000000005,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57099999999999995,"15min":0.57299999999999995},"min":{"1min":0.45500000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.69499999999999995,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.60199999999999998}]},{"osd":5,"last update":"Wed May 15 15:29:44 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.748}]},{"osd":6,"last update":"Wed May 15 15:30:05 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.70399999999999996}]},{"osd":7,"last update":"Wed May 15 15:30:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.65000000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724571,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:29:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.54700000000000004}]},{"osd":1,"last update":"Wed May 15 15:29:45 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.57999999999999996}]},{"osd":2,"last update":"Wed May 15 15:29:45 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.53100000000000003}]},{"osd":3,"last update":"Wed May 15 15:29:45 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.63100000000000001}]},{"osd":4,"last update":"Wed May 15 15:29:45 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.089,"5min":1.089,"15min":1.089},"last":0.438},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.51200000000000001}]},{"osd":6,"last update":"Wed May 15 15:30:07 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.47299999999999998}]},{"osd":7,"last update":"Wed May 15 15:30:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.68500000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-15T15:30:52.240 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph pg dump --format=json 2024-05-15T15:30:52.486 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 podman[47156]: 2024-05-15 15:30:52.315124967 +0000 UTC m=+1.129029294 container init 5851db4fb3e109f623095ed98cae0656f073c10c718f26f55995bfbd70e6c104 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 podman[47156]: 2024-05-15 15:30:52.319558707 +0000 UTC m=+1.133463045 container start 5851db4fb3e109f623095ed98cae0656f073c10c718f26f55995bfbd70e6c104 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:30:52.332Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:30:52.332Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:30:52.334Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.43 port=9094 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:30:52.335Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:30:52.361Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:30:52.361Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:30:52.363Z caller=main.go:518 msg=Listening address=:9093 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:30:52.364Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 bash[47156]: 5851db4fb3e109f623095ed98cae0656f073c10c718f26f55995bfbd70e6c104 2024-05-15T15:30:52.487 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:52 smithi043 systemd[1]: Started Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:30:52.488 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:52.740 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:30:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:30:52] "GET /metrics HTTP/1.1" 200 191472 "" "Prometheus/2.33.4" 2024-05-15T15:30:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:52 smithi053 ceph-mon[25205]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-15T15:30:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:52 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:52 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:52 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:52 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:30:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:52 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:53.085 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[29127]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-15T15:30:53.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:53.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:53.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:53.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:30:53.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[24430]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-15T15:30:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:30:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:52 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:53.938 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:53 smithi043 ceph-mon[24430]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:53.939 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:53 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:30:53.939 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:53 smithi043 ceph-mon[24430]: Deploying daemon grafana.a on smithi053 2024-05-15T15:30:53.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:53 smithi043 ceph-mon[29127]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:53.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:53 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:30:53.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:53 smithi043 ceph-mon[29127]: Deploying daemon grafana.a on smithi053 2024-05-15T15:30:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:53 smithi053 ceph-mon[25205]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:53 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:30:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:53 smithi053 ceph-mon[25205]: Deploying daemon grafana.a on smithi053 2024-05-15T15:30:54.327 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:30:54.329 INFO:teuthology.orchestra.run.smithi043.stderr:dumped all 2024-05-15T15:30:54.585 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:30:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:30:54.335Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000138898s 2024-05-15T15:30:54.949 INFO:teuthology.orchestra.run.smithi043.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-05-15T15:30:54.272589+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48972,"kb_used_data":6156,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912244,"statfs":{"total":767960285184,"available":767910137856,"internally_reserved":0,"allocated":6303744,"data_stored":3511261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-05-15T15:29:44.339514+0000","last_change":"2024-05-15T15:29:32.577058+0000","last_active":"2024-05-15T15:29:44.339514+0000","last_peered":"2024-05-15T15:29:44.339514+0000","last_clean":"2024-05-15T15:29:44.339514+0000","last_became_active":"2024-05-15T15:29:32.576685+0000","last_became_peered":"2024-05-15T15:29:32.576685+0000","last_unstale":"2024-05-15T15:29:44.339514+0000","last_undegraded":"2024-05-15T15:29:44.339514+0000","last_fullsized":"2024-05-15T15:29:44.339514+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-15T15:27:32.692157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-15T15:27:32.692157+0000","last_clean_scrub_stamp":"2024-05-15T15:27:32.692157+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-16T16:20:34.823822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201298,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.72599999999999998}]},{"osd":1,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.58799999999999997}]},{"osd":2,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.53500000000000003}]},{"osd":3,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.61799999999999999}]},{"osd":4,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.60699999999999998}]},{"osd":5,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.79400000000000004}]},{"osd":6,"last update":"Wed May 15 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.745}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":769050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.55300000000000005}]},{"osd":1,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.58499999999999996}]},{"osd":2,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.628},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.51700000000000002}]},{"osd":3,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.57099999999999995}]},{"osd":4,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.42699999999999999}]},{"osd":5,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.46899999999999997}]},{"osd":7,"last update":"Wed May 15 15:30:32 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.60499999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:16 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.35999999999999999,"15min":0.34100000000000003},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.371},{"interface":"front","average":{"1min":0.44,"5min":0.38700000000000001,"15min":0.35899999999999999},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.46200000000000002}]},{"osd":2,"last update":"Wed May 15 15:30:38 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.44700000000000001,"15min":0.439},"min":{"1min":0.314,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40300000000000002},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.41699999999999998}]},{"osd":3,"last update":"Wed May 15 15:30:01 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.5,"15min":0.497},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.44}]},{"osd":4,"last update":"Wed May 15 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.64400000000000002,"15min":0.65200000000000002},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.748,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79200000000000004,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.52700000000000002}]},{"osd":5,"last update":"Wed May 15 15:30:45 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.61299999999999999,"15min":0.61099999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.82599999999999996,"5min":0.878,"15min":0.878},"last":0.496},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.624,"15min":0.626},"min":{"1min":0.44,"5min":0.437,"15min":0.437},"max":{"1min":0.83599999999999997,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.51100000000000001}]},{"osd":6,"last update":"Wed May 15 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.57299999999999995}]},{"osd":7,"last update":"Wed May 15 15:30:30 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.61899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed May 15 15:30:10 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.36499999999999999,"15min":0.34699999999999998},"min":{"1min":0.30599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.59299999999999997,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.40200000000000002,"15min":0.38800000000000001},"min":{"1min":0.28299999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.34499999999999997}]},{"osd":2,"last update":"Wed May 15 15:30:35 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.39900000000000002,"15min":0.39100000000000001},"min":{"1min":0.30199999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.59299999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.40500000000000003,"15min":0.40699999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54500000000000004,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.27100000000000002}]},{"osd":3,"last update":"Wed May 15 15:30:02 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46500000000000002,"15min":0.46600000000000003},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.47999999999999998,"15min":0.48499999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.625,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.439}]},{"osd":4,"last update":"Wed May 15 15:30:22 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.59999999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76100000000000001,"5min":1.018,"15min":1.018},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.59899999999999998,"15min":0.60499999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74299999999999999,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.59299999999999997}]},{"osd":5,"last update":"Wed May 15 15:30:46 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.59299999999999997,"15min":0.58999999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.745,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.624,"15min":0.626},"min":{"1min":0.51300000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.81999999999999995,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.72099999999999997}]},{"osd":6,"last update":"Wed May 15 15:30:09 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.754}]},{"osd":7,"last update":"Wed May 15 15:30:29 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.66300000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:39 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56200000000000006,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.44},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.39200000000000002,"15min":0.40100000000000002},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.48499999999999999,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.51900000000000002}]},{"osd":1,"last update":"Wed May 15 15:30:39 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.40500000000000003,"15min":0.41599999999999998},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.61399999999999999,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.436,"15min":0.443},"min":{"1min":0.28299999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.50800000000000001,"5min":0.75,"15min":0.75},"last":0.54100000000000004}]},{"osd":3,"last update":"Wed May 15 15:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.48199999999999998,"15min":0.48599999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.52200000000000002,"15min":0.52800000000000002},"min":{"1min":0.39300000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.66500000000000004,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.59499999999999997}]},{"osd":4,"last update":"Wed May 15 15:30:28 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.80300000000000005,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.65300000000000002}]},{"osd":5,"last update":"Wed May 15 15:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.59299999999999997,"15min":0.58699999999999997},"min":{"1min":0.47099999999999997,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.57499999999999996,"15min":0.57599999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.82999999999999996}]},{"osd":6,"last update":"Wed May 15 15:30:06 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.93999999999999995},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.046,"5min":1.046,"15min":1.046},"last":0.86699999999999999}]},{"osd":7,"last update":"Wed May 15 15:30:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.93000000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149732,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:29:58 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.40899999999999997,"15min":0.41199999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.60299999999999998,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.443,"15min":0.44700000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.61199999999999999,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.37}]},{"osd":1,"last update":"Wed May 15 15:29:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.442,"15min":0.442},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.497},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.433,"15min":0.435},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.58899999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.51300000000000001}]},{"osd":2,"last update":"Wed May 15 15:29:58 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.47399999999999998,"15min":0.47999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.59199999999999997,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.46600000000000003,"15min":0.46999999999999997},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.55000000000000004,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.60799999999999998}]},{"osd":4,"last update":"Wed May 15 15:30:21 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.623,"15min":0.61799999999999999},"min":{"1min":0.46000000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.63300000000000001,"15min":0.629},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.70899999999999996}]},{"osd":5,"last update":"Wed May 15 15:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.61699999999999999,"15min":0.60199999999999998},"min":{"1min":0.47999999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.266,"5min":1.266,"15min":1.266},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.64000000000000001,"15min":0.63100000000000001},"min":{"1min":0.434,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.624}]},{"osd":6,"last update":"Wed May 15 15:30:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.58399999999999996}]},{"osd":7,"last update":"Wed May 15 15:30:29 2024","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.57199999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.58399999999999996,"15min":0.59599999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69899999999999995,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.53100000000000003,"15min":0.53400000000000003},"min":{"1min":0.39200000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.53200000000000003}]},{"osd":1,"last update":"Wed May 15 15:30:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.65900000000000003,"5min":0.746,"15min":0.746},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.56799999999999995,"15min":0.57099999999999995},"min":{"1min":0.36399999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.75900000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.64200000000000002}]},{"osd":2,"last update":"Wed May 15 15:30:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.39200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.70899999999999996,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.57799999999999996,"15min":0.58399999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66900000000000004,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.628}]},{"osd":3,"last update":"Wed May 15 15:30:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.57699999999999996,"15min":0.58299999999999996},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68500000000000005,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57099999999999995,"15min":0.57299999999999995},"min":{"1min":0.45500000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.69499999999999995,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56000000000000005}]},{"osd":5,"last update":"Wed May 15 15:30:48 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.46100000000000002,"15min":0.45800000000000002},"min":{"1min":0.32900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68200000000000005,"5min":0.746,"15min":0.746},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.47099999999999997,"15min":0.46500000000000002},"min":{"1min":0.34300000000000003,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.71499999999999997}]},{"osd":6,"last update":"Wed May 15 15:30:05 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.76400000000000001}]},{"osd":7,"last update":"Wed May 15 15:30:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.65700000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 15 15:30:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.68799999999999994,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.56999999999999995,"15min":0.57699999999999996},"min":{"1min":0.34200000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.65700000000000003,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.56899999999999995}]},{"osd":1,"last update":"Wed May 15 15:30:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56699999999999995,"15min":0.57099999999999995},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.67700000000000005,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.58999999999999997,"15min":0.59899999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67800000000000005,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.54200000000000004}]},{"osd":2,"last update":"Wed May 15 15:30:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.65600000000000003,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.625},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.59999999999999998,"15min":0.60499999999999998},"min":{"1min":0.42899999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.65800000000000003,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.53000000000000003}]},{"osd":3,"last update":"Wed May 15 15:30:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.56499999999999995,"15min":0.56899999999999995},"min":{"1min":0.38300000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67200000000000004,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.58399999999999996,"15min":0.58599999999999997},"min":{"1min":0.45200000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73999999999999999,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.57399999999999995}]},{"osd":4,"last update":"Wed May 15 15:30:47 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.441,"15min":0.432},"min":{"1min":0.318,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.68799999999999994,"5min":1.089,"15min":1.089},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.45800000000000002,"15min":0.44800000000000001},"min":{"1min":0.32300000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.69299999999999995,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.55100000000000005}]},{"osd":6,"last update":"Wed May 15 15:30:07 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.499},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.61399999999999999}]},{"osd":7,"last update":"Wed May 15 15:30:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.58899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-15T15:30:54.951 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-05-15T15:30:54.951 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-05-15T15:30:54.951 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-05-15T15:30:54.952 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- ceph health --format=json 2024-05-15T15:30:55.178 INFO:teuthology.orchestra.run.smithi043.stderr:Inferring config /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/mon.a/config 2024-05-15T15:30:55.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:55 smithi043 ceph-mon[24430]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:55.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:55 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:55.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:55 smithi043 ceph-mon[24430]: from='client.24502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-15T15:30:55.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:55 smithi043 ceph-mon[29127]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:55.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:55 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:55.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:55 smithi043 ceph-mon[29127]: from='client.24502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-15T15:30:55.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:55 smithi053 ceph-mon[25205]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:55.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:55 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:30:55.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:55 smithi053 ceph-mon[25205]: from='client.24502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-15T15:30:56.993 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:30:56.994 INFO:teuthology.orchestra.run.smithi043.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-05-15T15:30:57.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:30:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:30:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:30:57.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:57 smithi043 ceph-mon[24430]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:57.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:57 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/376953748' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-15T15:30:57.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:57 smithi043 ceph-mon[29127]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:57.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:57 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/376953748' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-15T15:30:57.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:57 smithi053 ceph-mon[25205]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:57 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/376953748' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-15T15:30:57.655 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-05-15T15:30:57.656 INFO:tasks.cephadm:Setup complete, yielding 2024-05-15T15:30:57.656 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-15T15:30:57.667 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi043.front.sepia.ceph.com 2024-05-15T15:30:57.667 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-05-15T15:30:59.486 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:30:59 smithi043 ceph-mon[24430]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:59.486 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:30:59 smithi043 ceph-mon[29127]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:30:59.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:30:59 smithi053 ceph-mon[25205]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:01.369 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:31:01.369 INFO:teuthology.orchestra.run.smithi043.stdout: "id": "f42e0f68-c58c-4318-ac20-8a37fc5f23cb", 2024-05-15T15:31:01.370 INFO:teuthology.orchestra.run.smithi043.stdout: "name": "r", 2024-05-15T15:31:01.370 INFO:teuthology.orchestra.run.smithi043.stdout: "current_period": "1f502b76-4f5e-4b7e-b73c-dec20cdab754", 2024-05-15T15:31:01.370 INFO:teuthology.orchestra.run.smithi043.stdout: "epoch": 1 2024-05-15T15:31:01.370 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:31:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:01 smithi053 ceph-mon[25205]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:01 smithi053 ceph-mon[25205]: osdmap e61: 8 total, 8 up, 8 in 2024-05-15T15:31:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:01 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3026790784' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-15T15:31:01.673 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:01 smithi043 ceph-mon[24430]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:01.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:01 smithi043 ceph-mon[24430]: osdmap e61: 8 total, 8 up, 8 in 2024-05-15T15:31:01.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:01 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3026790784' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-15T15:31:01.674 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:01 smithi043 ceph-mon[29127]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:01.674 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:01 smithi043 ceph-mon[29127]: osdmap e61: 8 total, 8 up, 8 in 2024-05-15T15:31:01.674 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:01 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3026790784' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-05-15T15:31:01.952 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-05-15T15:31:02.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:02 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3026790784' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-15T15:31:02.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:02 smithi043 ceph-mon[29127]: osdmap e62: 8 total, 8 up, 8 in 2024-05-15T15:31:02.586 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:31:02.338Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002326601s 2024-05-15T15:31:02.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:02 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3026790784' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-15T15:31:02.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:02 smithi043 ceph-mon[24430]: osdmap e62: 8 total, 8 up, 8 in 2024-05-15T15:31:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:02 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3026790784' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-05-15T15:31:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:02 smithi053 ceph-mon[25205]: osdmap e62: 8 total, 8 up, 8 in 2024-05-15T15:31:02.938 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:31:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:02] "GET /metrics HTTP/1.1" 200 191476 "" "Prometheus/2.33.4" 2024-05-15T15:31:03.443 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:03 smithi043 ceph-mon[24430]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:03.443 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:03 smithi043 ceph-mon[24430]: osdmap e63: 8 total, 8 up, 8 in 2024-05-15T15:31:03.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:03 smithi043 ceph-mon[29127]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:03.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:03 smithi043 ceph-mon[29127]: osdmap e63: 8 total, 8 up, 8 in 2024-05-15T15:31:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:03 smithi053 ceph-mon[25205]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:03 smithi053 ceph-mon[25205]: osdmap e63: 8 total, 8 up, 8 in 2024-05-15T15:31:03.640 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:31:03.640 INFO:teuthology.orchestra.run.smithi043.stdout: "id": "8da47f4f-7b82-4a2a-85d6-a4cb4f761ec5", 2024-05-15T15:31:03.640 INFO:teuthology.orchestra.run.smithi043.stdout: "name": "default", 2024-05-15T15:31:03.640 INFO:teuthology.orchestra.run.smithi043.stdout: "api_name": "default", 2024-05-15T15:31:03.640 INFO:teuthology.orchestra.run.smithi043.stdout: "is_master": "true", 2024-05-15T15:31:03.640 INFO:teuthology.orchestra.run.smithi043.stdout: "endpoints": [], 2024-05-15T15:31:03.640 INFO:teuthology.orchestra.run.smithi043.stdout: "hostnames": [], 2024-05-15T15:31:03.640 INFO:teuthology.orchestra.run.smithi043.stdout: "hostnames_s3website": [], 2024-05-15T15:31:03.641 INFO:teuthology.orchestra.run.smithi043.stdout: "master_zone": "", 2024-05-15T15:31:03.641 INFO:teuthology.orchestra.run.smithi043.stdout: "zones": [], 2024-05-15T15:31:03.641 INFO:teuthology.orchestra.run.smithi043.stdout: "placement_targets": [], 2024-05-15T15:31:03.641 INFO:teuthology.orchestra.run.smithi043.stdout: "default_placement": "", 2024-05-15T15:31:03.641 INFO:teuthology.orchestra.run.smithi043.stdout: "realm_id": "f42e0f68-c58c-4318-ac20-8a37fc5f23cb", 2024-05-15T15:31:03.641 INFO:teuthology.orchestra.run.smithi043.stdout: "sync_policy": { 2024-05-15T15:31:03.641 INFO:teuthology.orchestra.run.smithi043.stdout: "groups": [] 2024-05-15T15:31:03.641 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:31:03.641 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:31:04.140 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-05-15T15:31:05.349 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:05 smithi053 ceph-mon[25205]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:05.394 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:05 smithi043 ceph-mon[24430]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:05.394 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:05 smithi043 ceph-mon[29127]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:05.852 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "id": "a787adef-8474-4c39-8a7e-6eaf4453d157", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "name": "z", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "domain_root": "z.rgw.meta:root", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "control_pool": "z.rgw.control", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "gc_pool": "z.rgw.log:gc", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "lc_pool": "z.rgw.log:lc", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "log_pool": "z.rgw.log", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "roles_pool": "z.rgw.meta:roles", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-05-15T15:31:05.853 INFO:teuthology.orchestra.run.smithi043.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "otp_pool": "z.rgw.otp", 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "system_key": { 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "access_key": "", 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "secret_key": "" 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "placement_pools": [ 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: { 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "key": "default-placement", 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "val": { 2024-05-15T15:31:05.854 INFO:teuthology.orchestra.run.smithi043.stdout: "index_pool": "z.rgw.buckets.index", 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: "storage_classes": { 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: "STANDARD": { 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: "data_pool": "z.rgw.buckets.data" 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: "index_type": 0 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: "realm_id": "f42e0f68-c58c-4318-ac20-8a37fc5f23cb", 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout: "notif_pool": "z.rgw.log:notif" 2024-05-15T15:31:05.855 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:31:06.531 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-05-15T15:31:07.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:31:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:31:07.581 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:07 smithi053 ceph-mon[25205]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-05-15T15:31:07.585 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:07 smithi043 ceph-mon[29127]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-05-15T15:31:07.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:07 smithi043 ceph-mon[24430]: pgmap v47: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-05-15T15:31:08.098 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-05-15T15:31:08.098 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-05-15T15:31:08.380 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-05-15T15:31:08.380 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-05-15T15:31:08.380 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-05-15T15:31:08.380 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-05-15T15:31:08.380 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=221.499646ms 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-05-15T15:31:08.381 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-05-15T15:31:08.382 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:31:08+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-05-15T15:31:09.130 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 bash[41139]: 3e2dc1d42275ddc9ca3498b06ba358d44716633b9273b58a1e1444f081162246 2024-05-15T15:31:09.130 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:31:08 smithi053 systemd[1]: Started Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:31:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[24430]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-05-15T15:31:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[24430]: osdmap e64: 8 total, 8 up, 8 in 2024-05-15T15:31:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/4249999505' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-15T15:31:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[24430]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-15T15:31:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[29127]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-05-15T15:31:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[29127]: osdmap e64: 8 total, 8 up, 8 in 2024-05-15T15:31:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/4249999505' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-15T15:31:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[29127]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-15T15:31:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:09 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:09 smithi053 ceph-mon[25205]: pgmap v48: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-05-15T15:31:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:09 smithi053 ceph-mon[25205]: osdmap e64: 8 total, 8 up, 8 in 2024-05-15T15:31:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:09 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/4249999505' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-15T15:31:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:09 smithi053 ceph-mon[25205]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-05-15T15:31:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:09 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:09 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:09 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:09 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:09 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:11.051 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:10 smithi043 ceph-mon[24430]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-15T15:31:11.052 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:10 smithi043 ceph-mon[24430]: osdmap e65: 8 total, 8 up, 8 in 2024-05-15T15:31:11.052 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:10 smithi043 ceph-mon[29127]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-15T15:31:11.052 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:10 smithi043 ceph-mon[29127]: osdmap e65: 8 total, 8 up, 8 in 2024-05-15T15:31:11.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:10 smithi053 ceph-mon[25205]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-05-15T15:31:11.093 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:10 smithi053 ceph-mon[25205]: osdmap e65: 8 total, 8 up, 8 in 2024-05-15T15:31:12.004 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:11 smithi053 ceph-mon[25205]: pgmap v51: 65 pgs: 15 unknown, 50 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-05-15T15:31:12.004 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:11 smithi053 ceph-mon[25205]: osdmap e66: 8 total, 8 up, 8 in 2024-05-15T15:31:12.004 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:11 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/4249999505' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-15T15:31:12.004 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:11 smithi053 ceph-mon[25205]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-15T15:31:12.004 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:11 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:12.004 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:11 smithi053 ceph-mon[25205]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-15T15:31:12.004 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:11 smithi053 ceph-mon[25205]: osdmap e67: 8 total, 8 up, 8 in 2024-05-15T15:31:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[29127]: pgmap v51: 65 pgs: 15 unknown, 50 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-05-15T15:31:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[29127]: osdmap e66: 8 total, 8 up, 8 in 2024-05-15T15:31:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/4249999505' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-15T15:31:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[29127]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-15T15:31:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[29127]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-15T15:31:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[29127]: osdmap e67: 8 total, 8 up, 8 in 2024-05-15T15:31:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[24430]: pgmap v51: 65 pgs: 15 unknown, 50 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-05-15T15:31:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[24430]: osdmap e66: 8 total, 8 up, 8 in 2024-05-15T15:31:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/4249999505' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-15T15:31:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[24430]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-05-15T15:31:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[24430]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-05-15T15:31:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:11 smithi043 ceph-mon[24430]: osdmap e67: 8 total, 8 up, 8 in 2024-05-15T15:31:13.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:31:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:12] "GET /metrics HTTP/1.1" 200 191476 "" "Prometheus/2.33.4" 2024-05-15T15:31:14.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:13 smithi043 ceph-mon[24430]: pgmap v54: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-05-15T15:31:14.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:13 smithi043 ceph-mon[24430]: osdmap e68: 8 total, 8 up, 8 in 2024-05-15T15:31:14.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:13 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-15T15:31:14.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:13 smithi043 ceph-mon[29127]: pgmap v54: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-05-15T15:31:14.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:13 smithi043 ceph-mon[29127]: osdmap e68: 8 total, 8 up, 8 in 2024-05-15T15:31:14.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:13 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-15T15:31:14.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:13 smithi053 ceph-mon[25205]: pgmap v54: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-05-15T15:31:14.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:13 smithi053 ceph-mon[25205]: osdmap e68: 8 total, 8 up, 8 in 2024-05-15T15:31:14.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:13 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-05-15T15:31:15.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-15T15:31:15.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[29127]: osdmap e69: 8 total, 8 up, 8 in 2024-05-15T15:31:15.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-15T15:31:15.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:15.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:15.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-15T15:31:15.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[29127]: osdmap e70: 8 total, 8 up, 8 in 2024-05-15T15:31:15.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-15T15:31:15.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-15T15:31:15.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[24430]: osdmap e69: 8 total, 8 up, 8 in 2024-05-15T15:31:15.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-15T15:31:15.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:15.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:15.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-15T15:31:15.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[24430]: osdmap e70: 8 total, 8 up, 8 in 2024-05-15T15:31:15.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:14 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-15T15:31:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:14 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-05-15T15:31:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:14 smithi053 ceph-mon[25205]: osdmap e69: 8 total, 8 up, 8 in 2024-05-15T15:31:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:14 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-05-15T15:31:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:14 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:14 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:14 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-05-15T15:31:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:14 smithi053 ceph-mon[25205]: osdmap e70: 8 total, 8 up, 8 in 2024-05-15T15:31:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:14 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-05-15T15:31:15.820 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:15 smithi043 ceph-mon[24430]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-15T15:31:15.820 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:15 smithi043 ceph-mon[24430]: Reconfiguring daemon alertmanager.a on smithi043 2024-05-15T15:31:15.820 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:15 smithi043 ceph-mon[24430]: pgmap v57: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 6 op/s 2024-05-15T15:31:15.820 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:15 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-15T15:31:15.820 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:15 smithi043 ceph-mon[24430]: osdmap e71: 8 total, 8 up, 8 in 2024-05-15T15:31:15.820 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:15 smithi043 ceph-mon[29127]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-15T15:31:15.820 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:15 smithi043 ceph-mon[29127]: Reconfiguring daemon alertmanager.a on smithi043 2024-05-15T15:31:15.821 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:15 smithi043 ceph-mon[29127]: pgmap v57: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 6 op/s 2024-05-15T15:31:15.821 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:15 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-15T15:31:15.821 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:15 smithi043 ceph-mon[29127]: osdmap e71: 8 total, 8 up, 8 in 2024-05-15T15:31:15.821 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:31:15.821 INFO:teuthology.orchestra.run.smithi043.stdout: "id": "9bd62be8-d37f-456b-83de-9194a87f1f8b", 2024-05-15T15:31:15.821 INFO:teuthology.orchestra.run.smithi043.stdout: "epoch": 1, 2024-05-15T15:31:15.821 INFO:teuthology.orchestra.run.smithi043.stdout: "predecessor_uuid": "1f502b76-4f5e-4b7e-b73c-dec20cdab754", 2024-05-15T15:31:15.821 INFO:teuthology.orchestra.run.smithi043.stdout: "sync_status": [], 2024-05-15T15:31:15.821 INFO:teuthology.orchestra.run.smithi043.stdout: "period_map": { 2024-05-15T15:31:15.821 INFO:teuthology.orchestra.run.smithi043.stdout: "id": "9bd62be8-d37f-456b-83de-9194a87f1f8b", 2024-05-15T15:31:15.821 INFO:teuthology.orchestra.run.smithi043.stdout: "zonegroups": [ 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: { 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "id": "8da47f4f-7b82-4a2a-85d6-a4cb4f761ec5", 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "name": "default", 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "api_name": "default", 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "is_master": "true", 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "endpoints": [], 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "hostnames": [], 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "hostnames_s3website": [], 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "master_zone": "a787adef-8474-4c39-8a7e-6eaf4453d157", 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "zones": [ 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: { 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "id": "a787adef-8474-4c39-8a7e-6eaf4453d157", 2024-05-15T15:31:15.822 INFO:teuthology.orchestra.run.smithi043.stdout: "name": "z", 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "endpoints": [], 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "log_meta": "false", 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "log_data": "false", 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "bucket_index_max_shards": 11, 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "read_only": "false", 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "tier_type": "", 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "sync_from_all": "true", 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "sync_from": [], 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "redirect_zone": "" 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "placement_targets": [ 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: { 2024-05-15T15:31:15.823 INFO:teuthology.orchestra.run.smithi043.stdout: "name": "default-placement", 2024-05-15T15:31:15.824 INFO:teuthology.orchestra.run.smithi043.stdout: "tags": [], 2024-05-15T15:31:15.824 INFO:teuthology.orchestra.run.smithi043.stdout: "storage_classes": [ 2024-05-15T15:31:15.824 INFO:teuthology.orchestra.run.smithi043.stdout: "STANDARD" 2024-05-15T15:31:15.824 INFO:teuthology.orchestra.run.smithi043.stdout: ] 2024-05-15T15:31:15.824 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:31:15.824 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T15:31:15.824 INFO:teuthology.orchestra.run.smithi043.stdout: "default_placement": "default-placement", 2024-05-15T15:31:15.824 INFO:teuthology.orchestra.run.smithi043.stdout: "realm_id": "f42e0f68-c58c-4318-ac20-8a37fc5f23cb", 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: "sync_policy": { 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: "groups": [] 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: "short_zone_ids": [ 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: { 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: "key": "a787adef-8474-4c39-8a7e-6eaf4453d157", 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: "val": 3999329280 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:31:15.825 INFO:teuthology.orchestra.run.smithi043.stdout: ] 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: "master_zonegroup": "8da47f4f-7b82-4a2a-85d6-a4cb4f761ec5", 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: "master_zone": "a787adef-8474-4c39-8a7e-6eaf4453d157", 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: "period_config": { 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: "bucket_quota": { 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: "enabled": false, 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: "check_on_raw": false, 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: "max_size": -1, 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: "max_size_kb": 0, 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: "max_objects": -1 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:31:15.826 INFO:teuthology.orchestra.run.smithi043.stdout: "user_quota": { 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "enabled": false, 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "check_on_raw": false, 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "max_size": -1, 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "max_size_kb": 0, 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "max_objects": -1 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "user_ratelimit": { 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "max_read_ops": 0, 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "max_write_ops": 0, 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "max_read_bytes": 0, 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "max_write_bytes": 0, 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "enabled": false 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:31:15.827 INFO:teuthology.orchestra.run.smithi043.stdout: "bucket_ratelimit": { 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "max_read_ops": 0, 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "max_write_ops": 0, 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "max_read_bytes": 0, 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "max_write_bytes": 0, 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "enabled": false 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "anonymous_ratelimit": { 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "max_read_ops": 0, 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "max_write_ops": 0, 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "max_read_bytes": 0, 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "max_write_bytes": 0, 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: "enabled": false 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:31:15.828 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:31:15.829 INFO:teuthology.orchestra.run.smithi043.stdout: "realm_id": "f42e0f68-c58c-4318-ac20-8a37fc5f23cb", 2024-05-15T15:31:15.829 INFO:teuthology.orchestra.run.smithi043.stdout: "realm_name": "r", 2024-05-15T15:31:15.829 INFO:teuthology.orchestra.run.smithi043.stdout: "realm_epoch": 2 2024-05-15T15:31:15.829 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:31:16.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:15 smithi053 ceph-mon[25205]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-15T15:31:16.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:15 smithi053 ceph-mon[25205]: Reconfiguring daemon alertmanager.a on smithi043 2024-05-15T15:31:16.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:15 smithi053 ceph-mon[25205]: pgmap v57: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 6 op/s 2024-05-15T15:31:16.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:15 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1894244897' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-05-15T15:31:16.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:15 smithi053 ceph-mon[25205]: osdmap e71: 8 total, 8 up, 8 in 2024-05-15T15:31:17.082 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-05-15T15:31:17.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:31:16 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:31:17.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:17 smithi043 ceph-mon[24430]: pgmap v60: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-05-15T15:31:17.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:17 smithi043 ceph-mon[29127]: pgmap v60: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-05-15T15:31:17.834 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:17 smithi043 systemd[1]: Stopping Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:31:18.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:17 smithi053 ceph-mon[25205]: pgmap v60: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-05-15T15:31:18.585 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:18 smithi043 bash[48818]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager.a" found: no such container 2024-05-15T15:31:19.022 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[47184]: level=info ts=2024-05-15T15:31:18.873Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-15T15:31:19.336 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:19 smithi043 podman[48847]: 2024-05-15 15:31:19.133832462 +0000 UTC m=+0.813336284 container died 5851db4fb3e109f623095ed98cae0656f073c10c718f26f55995bfbd70e6c104 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:19 smithi043 ceph-mon[29127]: pgmap v61: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 185 B/s rd, 370 B/s wr, 1 op/s 2024-05-15T15:31:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:19 smithi043 ceph-mon[24430]: pgmap v61: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 185 B/s rd, 370 B/s wr, 1 op/s 2024-05-15T15:31:19.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:19 smithi043 podman[48847]: 2024-05-15 15:31:19.566617698 +0000 UTC m=+1.246121536 container cleanup 5851db4fb3e109f623095ed98cae0656f073c10c718f26f55995bfbd70e6c104 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:19.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:19 smithi043 bash[48847]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a 2024-05-15T15:31:19.837 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:19 smithi043 podman[48881]: 2024-05-15 15:31:19.725872638 +0000 UTC m=+0.817568589 container remove 5851db4fb3e109f623095ed98cae0656f073c10c718f26f55995bfbd70e6c104 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:20.125 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:19 smithi043 podman[48881]: 2024-05-15 15:31:19.835738835 +0000 UTC m=+0.927434808 volume remove c9d288a5cf730292ba58970e8581c20662971abd54b9c9e334d1416b8b36c890 2024-05-15T15:31:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:19 smithi053 ceph-mon[25205]: pgmap v61: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 185 B/s rd, 370 B/s wr, 1 op/s 2024-05-15T15:31:20.306 INFO:teuthology.orchestra.run.smithi043.stdout:Scheduled rgw.foo update... 2024-05-15T15:31:20.377 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:20 smithi043 bash[48900]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager.a" found: no such container 2024-05-15T15:31:20.377 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:20 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-15T15:31:20.377 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:20 smithi043 systemd[1]: Stopped Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:31:20.377 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:20 smithi043 systemd[1]: Starting Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:31:20.656 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:20 smithi043 podman[48985]: 2024-05-15 15:31:20.27890724 +0000 UTC m=+0.018367980 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-05-15T15:31:20.656 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:20 smithi043 podman[48985]: 2024-05-15 15:31:20.39586668 +0000 UTC m=+0.135327406 volume create bab6f291297a988433f59eda2493190701b25f779867e43cecc3aefa660afcfb 2024-05-15T15:31:20.656 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:20 smithi043 podman[48985]: 2024-05-15 15:31:20.655373063 +0000 UTC m=+0.394833788 container create c7485334fa8cac3eef23551d6b5ef8b1c69e93d770ba1f6ab7be112abb90ce47 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:21.340 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph osd pool create foo' 2024-05-15T15:31:21.555 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:21 smithi043 ceph-mon[29127]: pgmap v62: 129 pgs: 129 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 784 B/s wr, 2 op/s 2024-05-15T15:31:21.555 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:21 smithi043 ceph-mon[29127]: from='client.24574 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:31:21.555 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:21 smithi043 ceph-mon[29127]: Saving service rgw.foo spec with placement count:2 2024-05-15T15:31:21.555 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:21 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:21.556 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-mon[24430]: pgmap v62: 129 pgs: 129 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 784 B/s wr, 2 op/s 2024-05-15T15:31:21.556 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-mon[24430]: from='client.24574 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:31:21.556 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-mon[24430]: Saving service rgw.foo spec with placement count:2 2024-05-15T15:31:21.556 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:21.556 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 podman[48985]: 2024-05-15 15:31:21.45329696 +0000 UTC m=+1.192757684 container init c7485334fa8cac3eef23551d6b5ef8b1c69e93d770ba1f6ab7be112abb90ce47 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:21.556 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 podman[48985]: 2024-05-15 15:31:21.458058682 +0000 UTC m=+1.197519423 container start c7485334fa8cac3eef23551d6b5ef8b1c69e93d770ba1f6ab7be112abb90ce47 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:21.556 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:31:21.469Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-15T15:31:21.556 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:31:21.469Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-15T15:31:21.556 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:31:21.470Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.43 port=9094 2024-05-15T15:31:21.556 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:31:21.472Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-15T15:31:21.556 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:31:21.503Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-15T15:31:21.557 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:31:21.503Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-15T15:31:21.557 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:31:21.505Z caller=main.go:518 msg=Listening address=:9093 2024-05-15T15:31:21.557 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:31:21.505Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-15T15:31:21.557 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 bash[48985]: c7485334fa8cac3eef23551d6b5ef8b1c69e93d770ba1f6ab7be112abb90ce47 2024-05-15T15:31:21.622 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:21 smithi053 ceph-mon[25205]: pgmap v62: 129 pgs: 129 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 784 B/s wr, 2 op/s 2024-05-15T15:31:21.622 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:21 smithi053 ceph-mon[25205]: from='client.24574 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:31:21.622 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:21 smithi053 ceph-mon[25205]: Saving service rgw.foo spec with placement count:2 2024-05-15T15:31:21.622 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:21 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:21.835 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:21 smithi043 systemd[1]: Started Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:31:22.815 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:22 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:22.815 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:22 smithi043 ceph-mon[29127]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:31:22.815 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:22 smithi043 ceph-mon[29127]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:31:22.815 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:22 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:22.816 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:22 smithi043 ceph-mon[24430]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:31:22.816 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:22 smithi043 ceph-mon[24430]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:31:22.816 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:31:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:22] "GET /metrics HTTP/1.1" 200 196473 "" "Prometheus/2.33.4" 2024-05-15T15:31:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:22 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:22 smithi053 ceph-mon[25205]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:31:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:22 smithi053 ceph-mon[25205]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:31:23.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:23 smithi043 ceph-mon[29127]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-05-15T15:31:23.776 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:31:23.473Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000496145s 2024-05-15T15:31:23.776 INFO:teuthology.orchestra.run.smithi043.stderr:pool 'foo' created 2024-05-15T15:31:23.776 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:23 smithi043 ceph-mon[24430]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-05-15T15:31:24.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:23 smithi053 ceph-mon[25205]: pgmap v63: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-05-15T15:31:24.444 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'rbd pool init foo' 2024-05-15T15:31:24.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 systemd[1]: Stopping Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:31:24.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 bash[41829]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus.a" found: no such container 2024-05-15T15:31:24.952 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:24 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2865612783' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-15T15:31:24.953 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:24 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2865612783' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-15T15:31:24.953 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:24 smithi053 ceph-mon[25205]: osdmap e72: 8 total, 8 up, 8 in 2024-05-15T15:31:24.953 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:24 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:24.953 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.667Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-15T15:31:24.953 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.667Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-15T15:31:24.953 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.668Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-15T15:31:24.953 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.668Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-15T15:31:24.953 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.668Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-15T15:31:24.953 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.668Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-15T15:31:24.954 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.668Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-15T15:31:24.954 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.668Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-15T15:31:24.954 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.668Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-15T15:31:24.954 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.668Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-15T15:31:24.954 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.668Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-15T15:31:24.954 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[40193]: ts=2024-05-15T15:31:24.668Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-15T15:31:24.954 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:24 smithi053 podman[41840]: 2024-05-15 15:31:24.678015476 +0000 UTC m=+0.131909209 container died 0870489507f965174036d784810e2b458a087030ae5f90061549d49b5fd79fc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:24.967 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:24 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2865612783' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-15T15:31:24.967 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:24 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2865612783' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-15T15:31:24.967 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:24 smithi043 ceph-mon[29127]: osdmap e72: 8 total, 8 up, 8 in 2024-05-15T15:31:24.968 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:24 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:24.968 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:24 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2865612783' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-05-15T15:31:24.968 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:24 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2865612783' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-05-15T15:31:24.968 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:24 smithi043 ceph-mon[24430]: osdmap e72: 8 total, 8 up, 8 in 2024-05-15T15:31:24.968 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:24 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:25.303 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 podman[41840]: 2024-05-15 15:31:25.042418614 +0000 UTC m=+0.496312339 container cleanup 0870489507f965174036d784810e2b458a087030ae5f90061549d49b5fd79fc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:25.303 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 bash[41840]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a 2024-05-15T15:31:25.303 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 bash[41866]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus.a" found: no such container 2024-05-15T15:31:25.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 podman[41852]: 2024-05-15 15:31:25.301728492 +0000 UTC m=+0.624482703 container remove 0870489507f965174036d784810e2b458a087030ae5f90061549d49b5fd79fc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:25.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-15T15:31:25.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 systemd[1]: Stopped Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:31:25.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 systemd[1]: Starting Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:31:25.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 podman[41928]: 2024-05-15 15:31:25.54423989 +0000 UTC m=+0.111257695 container create 8e95fb39ec80e823bbfcda958858016688b5935a6df44f442d831e1041731b6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:25.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 podman[41928]: 2024-05-15 15:31:25.452321738 +0000 UTC m=+0.019339553 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-15T15:31:25.913 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:25 smithi053 ceph-mon[25205]: pgmap v65: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 841 B/s wr, 2 op/s 2024-05-15T15:31:25.913 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:25 smithi053 ceph-mon[25205]: osdmap e73: 8 total, 8 up, 8 in 2024-05-15T15:31:25.913 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 podman[41928]: 2024-05-15 15:31:25.763507387 +0000 UTC m=+0.330525188 container init 8e95fb39ec80e823bbfcda958858016688b5935a6df44f442d831e1041731b6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:25.913 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 podman[41928]: 2024-05-15 15:31:25.766677038 +0000 UTC m=+0.333694837 container start 8e95fb39ec80e823bbfcda958858016688b5935a6df44f442d831e1041731b6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:31:25.913 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.791Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-15T15:31:25.913 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.791Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.791Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.791Z caller=main.go:518 level=info host_details="(Linux 5.14.0-446.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024 x86_64 smithi053 (none))" 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.791Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.791Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.793Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.793Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.794Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.797Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.797Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.87µs 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:25.797Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 bash[41928]: 8e95fb39ec80e823bbfcda958858016688b5935a6df44f442d831e1041731b6e 2024-05-15T15:31:25.914 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:25 smithi053 systemd[1]: Started Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:31:26.021 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:25 smithi043 ceph-mon[24430]: pgmap v65: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 841 B/s wr, 2 op/s 2024-05-15T15:31:26.021 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:25 smithi043 ceph-mon[24430]: osdmap e73: 8 total, 8 up, 8 in 2024-05-15T15:31:26.021 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:25 smithi043 ceph-mon[29127]: pgmap v65: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 841 B/s wr, 2 op/s 2024-05-15T15:31:26.021 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:25 smithi043 ceph-mon[29127]: osdmap e73: 8 total, 8 up, 8 in 2024-05-15T15:31:26.380 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:26.005Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-05-15T15:31:26.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:26.005Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-05-15T15:31:26.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:26.005Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.114µs wal_replay_duration=208.570111ms total_replay_duration=208.612022ms 2024-05-15T15:31:26.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:26.007Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-15T15:31:26.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:26.007Z caller=main.go:947 level=info msg="TSDB started" 2024-05-15T15:31:26.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:26.007Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-15T15:31:26.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:26.024Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.372347ms db_storage=695ns remote_storage=1.226µs web_handler=366ns query_engine=831ns scrape=664.694µs scrape_sd=26.154µs notify=31.216µs notify_sd=9.892µs rules=16.407631ms 2024-05-15T15:31:26.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:31:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:31:26.024Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-15T15:31:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: osdmap e74: 8 total, 8 up, 8 in 2024-05-15T15:31:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:31:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:31:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.43:9093"}]: dispatch 2024-05-15T15:31:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.43:9093"}]: dispatch 2024-05-15T15:31:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.53:9095"}]: dispatch 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.53:9095"}]: dispatch 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.53:3000"}]: dispatch 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:26.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/49267476' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: osdmap e74: 8 total, 8 up, 8 in 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.43:9093"}]: dispatch 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.43:9093"}]: dispatch 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.53:9095"}]: dispatch 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.53:9095"}]: dispatch 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:31:26.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:31:26.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.53:3000"}]: dispatch 2024-05-15T15:31:26.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:26.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:26.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:26.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:26 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/49267476' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-15T15:31:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: osdmap e74: 8 total, 8 up, 8 in 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.43:9093"}]: dispatch 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.43:9093"}]: dispatch 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.53:9095"}]: dispatch 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.53:9095"}]: dispatch 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:31:26.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.53:3000"}]: dispatch 2024-05-15T15:31:26.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:26.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:26.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:26.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:26.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:26 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/49267476' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-05-15T15:31:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:27 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.53:3000"}]: dispatch 2024-05-15T15:31:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:27 smithi043 ceph-mon[24430]: pgmap v68: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 682 B/s wr, 1 op/s 2024-05-15T15:31:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:27 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/49267476' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-15T15:31:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:27 smithi043 ceph-mon[24430]: osdmap e75: 8 total, 8 up, 8 in 2024-05-15T15:31:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:27 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.53:3000"}]: dispatch 2024-05-15T15:31:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:27 smithi043 ceph-mon[29127]: pgmap v68: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 682 B/s wr, 1 op/s 2024-05-15T15:31:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:27 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/49267476' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-15T15:31:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:27 smithi043 ceph-mon[29127]: osdmap e75: 8 total, 8 up, 8 in 2024-05-15T15:31:28.129 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:27 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.53:3000"}]: dispatch 2024-05-15T15:31:28.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:27 smithi053 ceph-mon[25205]: pgmap v68: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 682 B/s wr, 1 op/s 2024-05-15T15:31:28.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:27 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/49267476' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-05-15T15:31:28.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:27 smithi053 ceph-mon[25205]: osdmap e75: 8 total, 8 up, 8 in 2024-05-15T15:31:29.274 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:28 smithi043 ceph-mon[24430]: osdmap e76: 8 total, 8 up, 8 in 2024-05-15T15:31:29.274 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:28 smithi043 ceph-mon[24430]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:29.274 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:28 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:29.274 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:28 smithi043 ceph-mon[29127]: osdmap e76: 8 total, 8 up, 8 in 2024-05-15T15:31:29.275 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:28 smithi043 ceph-mon[29127]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:29.275 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:28 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:29.348 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:28 smithi053 ceph-mon[25205]: osdmap e76: 8 total, 8 up, 8 in 2024-05-15T15:31:29.348 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:28 smithi053 ceph-mon[25205]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:29.349 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:28 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:29.611 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch apply iscsi foo u p' 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: osdmap e77: 8 total, 8 up, 8 in 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: Saving service rgw.foo spec with placement count:2 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:30.316 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[29127]: Deploying daemon rgw.foo.smithi043.fiueec on smithi043 2024-05-15T15:31:30.316 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: osdmap e77: 8 total, 8 up, 8 in 2024-05-15T15:31:30.316 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.316 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.316 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.316 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.316 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: Saving service rgw.foo spec with placement count:2 2024-05-15T15:31:30.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:30.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:30.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-15T15:31:30.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:30.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:29 smithi043 ceph-mon[24430]: Deploying daemon rgw.foo.smithi043.fiueec on smithi043 2024-05-15T15:31:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: osdmap e77: 8 total, 8 up, 8 in 2024-05-15T15:31:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: Saving service rgw.foo spec with placement count:2 2024-05-15T15:31:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:30.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:30.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-15T15:31:30.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:30.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:30.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:29 smithi053 ceph-mon[25205]: Deploying daemon rgw.foo.smithi043.fiueec on smithi043 2024-05-15T15:31:31.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:31 smithi043 ceph-mon[24430]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 454 B/s wr, 0 op/s 2024-05-15T15:31:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:31 smithi043 ceph-mon[29127]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 454 B/s wr, 0 op/s 2024-05-15T15:31:31.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:31 smithi053 ceph-mon[25205]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 454 B/s wr, 0 op/s 2024-05-15T15:31:31.754 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:31:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:31:31.475Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002644252s 2024-05-15T15:31:32.470 INFO:teuthology.orchestra.run.smithi043.stdout:Scheduled iscsi.foo update... 2024-05-15T15:31:32.795 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:31:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:32] "GET /metrics HTTP/1.1" 200 197899 "" "Prometheus/2.33.4" 2024-05-15T15:31:33.455 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'sleep 180' 2024-05-15T15:31:33.738 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:33 smithi053 ceph-mon[25205]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-05-15T15:31:33.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:33 smithi053 ceph-mon[25205]: from='client.24592 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:31:33.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:33 smithi053 ceph-mon[25205]: Saving service iscsi.foo spec with placement count:1 2024-05-15T15:31:33.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:33 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:33.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:33 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:33.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:33 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:33.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:33 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:33.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:33 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-15T15:31:33.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:33 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:33.739 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:33 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:33.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[29127]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-05-15T15:31:33.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[29127]: from='client.24592 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:31:33.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[29127]: Saving service iscsi.foo spec with placement count:1 2024-05-15T15:31:33.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:33.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:33.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:33.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:33.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[24430]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[24430]: from='client.24592 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[24430]: Saving service iscsi.foo spec with placement count:1 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-15T15:31:33.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:33.768 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:33 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:34.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:34 smithi053 ceph-mon[25205]: Deploying daemon rgw.foo.smithi053.vjmwpu on smithi053 2024-05-15T15:31:34.705 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:34 smithi043 ceph-mon[24430]: Deploying daemon rgw.foo.smithi053.vjmwpu on smithi053 2024-05-15T15:31:34.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:34 smithi043 ceph-mon[29127]: Deploying daemon rgw.foo.smithi053.vjmwpu on smithi053 2024-05-15T15:31:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[29127]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 279 B/s wr, 0 op/s 2024-05-15T15:31:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[29127]: osdmap e78: 8 total, 8 up, 8 in 2024-05-15T15:31:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:35.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[24430]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 279 B/s wr, 0 op/s 2024-05-15T15:31:35.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[24430]: osdmap e78: 8 total, 8 up, 8 in 2024-05-15T15:31:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:35 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:35 smithi053 ceph-mon[25205]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 279 B/s wr, 0 op/s 2024-05-15T15:31:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:35 smithi053 ceph-mon[25205]: osdmap e78: 8 total, 8 up, 8 in 2024-05-15T15:31:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:35 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:35 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:35 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:35 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:37.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:31:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:31:37.792 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:37 smithi043 ceph-mon[24430]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 4.4 KiB/s wr, 141 op/s 2024-05-15T15:31:37.793 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:37 smithi043 ceph-mon[29127]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 4.4 KiB/s wr, 141 op/s 2024-05-15T15:31:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:37 smithi053 ceph-mon[25205]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 4.4 KiB/s wr, 141 op/s 2024-05-15T15:31:39.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:39 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:39.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:39 smithi043 ceph-mon[24430]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 3.8 KiB/s wr, 121 op/s 2024-05-15T15:31:39.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:39 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:39.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:39 smithi043 ceph-mon[29127]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 3.8 KiB/s wr, 121 op/s 2024-05-15T15:31:39.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:39 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:39.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:39 smithi053 ceph-mon[25205]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 3.8 KiB/s wr, 121 op/s 2024-05-15T15:31:40.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:40 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:40.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:40 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:40 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:41.620 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:41 smithi043 ceph-mon[24430]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-05-15T15:31:41.621 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:41 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:41.621 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:41 smithi043 ceph-mon[29127]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-05-15T15:31:41.621 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:41 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:41 smithi053 ceph-mon[25205]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-05-15T15:31:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:41 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:42.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:31:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:42] "GET /metrics HTTP/1.1" 200 197899 "" "Prometheus/2.33.4" 2024-05-15T15:31:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[24430]: Checking dashboard <-> RGW credentials 2024-05-15T15:31:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[24430]: Checking pool "foo" exists for service iscsi.foo 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[24430]: Deploying daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[29127]: Checking dashboard <-> RGW credentials 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[29127]: Checking pool "foo" exists for service iscsi.foo 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:31:42.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:31:42.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-15T15:31:42.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:42.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:42 smithi043 ceph-mon[29127]: Deploying daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:31:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:42 smithi053 ceph-mon[25205]: Checking dashboard <-> RGW credentials 2024-05-15T15:31:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:42 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:42 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:42 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:42 smithi053 ceph-mon[25205]: Checking pool "foo" exists for service iscsi.foo 2024-05-15T15:31:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:42 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:31:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:42 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:31:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:42 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-05-15T15:31:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:42 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:42 smithi053 ceph-mon[25205]: Deploying daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:31:43.634 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:43 smithi043 ceph-mon[24430]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-05-15T15:31:43.634 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:43 smithi043 ceph-mon[24430]: mgrmap e20: y(active, since 118s), standbys: x 2024-05-15T15:31:43.634 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:43 smithi043 ceph-mon[29127]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-05-15T15:31:43.634 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:43 smithi043 ceph-mon[29127]: mgrmap e20: y(active, since 118s), standbys: x 2024-05-15T15:31:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:43 smithi053 ceph-mon[25205]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-05-15T15:31:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:43 smithi053 ceph-mon[25205]: mgrmap e20: y(active, since 118s), standbys: x 2024-05-15T15:31:44.676 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-15T15:31:44.676 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.676 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-15T15:31:44.676 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:44.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:31:44.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:31:44.679 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:44.679 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:44.679 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:44.679 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:31:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:31:44.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:31:44.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:44.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:44.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:44.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:45.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[29127]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-05-15T15:31:45.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-05-15T15:31:45.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-05-15T15:31:45.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-05-15T15:31:45.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-05-15T15:31:45.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[29127]: osdmap e79: 8 total, 8 up, 8 in 2024-05-15T15:31:45.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3958141778' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T15:31:45.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3269453152' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3688153742"}]: dispatch 2024-05-15T15:31:45.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3688153742"}]: dispatch 2024-05-15T15:31:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:45 smithi053 ceph-mon[25205]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-05-15T15:31:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:45 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-05-15T15:31:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:45 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-05-15T15:31:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:45 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-05-15T15:31:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:45 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-05-15T15:31:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:45 smithi053 ceph-mon[25205]: osdmap e79: 8 total, 8 up, 8 in 2024-05-15T15:31:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:45 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3958141778' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T15:31:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:45 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3269453152' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3688153742"}]: dispatch 2024-05-15T15:31:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:45 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3688153742"}]: dispatch 2024-05-15T15:31:46.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[24430]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 3.4 KiB/s wr, 212 op/s 2024-05-15T15:31:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-05-15T15:31:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-05-15T15:31:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-05-15T15:31:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-05-15T15:31:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[24430]: osdmap e79: 8 total, 8 up, 8 in 2024-05-15T15:31:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3958141778' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T15:31:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3269453152' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3688153742"}]: dispatch 2024-05-15T15:31:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:45 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3688153742"}]: dispatch 2024-05-15T15:31:46.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:46 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3688153742"}]': finished 2024-05-15T15:31:46.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:46 smithi043 ceph-mon[29127]: osdmap e80: 8 total, 8 up, 8 in 2024-05-15T15:31:46.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:46 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/4029531881' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3688153742"}]: dispatch 2024-05-15T15:31:46.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:46 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3688153742"}]: dispatch 2024-05-15T15:31:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:46 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3688153742"}]': finished 2024-05-15T15:31:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:46 smithi043 ceph-mon[24430]: osdmap e80: 8 total, 8 up, 8 in 2024-05-15T15:31:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:46 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/4029531881' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3688153742"}]: dispatch 2024-05-15T15:31:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:46 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3688153742"}]: dispatch 2024-05-15T15:31:46.842 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:46 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3688153742"}]': finished 2024-05-15T15:31:46.842 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:46 smithi053 ceph-mon[25205]: osdmap e80: 8 total, 8 up, 8 in 2024-05-15T15:31:46.842 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:46 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/4029531881' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3688153742"}]: dispatch 2024-05-15T15:31:46.842 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:46 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3688153742"}]: dispatch 2024-05-15T15:31:47.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:31:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:31:47.838 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:47 smithi053 ceph-mon[25205]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 511 B/s wr, 130 op/s 2024-05-15T15:31:47.838 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:47 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3688153742"}]': finished 2024-05-15T15:31:47.838 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:47 smithi053 ceph-mon[25205]: osdmap e81: 8 total, 8 up, 8 in 2024-05-15T15:31:47.838 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:47 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2803988765' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2314145657"}]: dispatch 2024-05-15T15:31:47.838 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:47 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2314145657"}]: dispatch 2024-05-15T15:31:47.853 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:47 smithi043 ceph-mon[24430]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 511 B/s wr, 130 op/s 2024-05-15T15:31:47.853 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:47 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3688153742"}]': finished 2024-05-15T15:31:47.853 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:47 smithi043 ceph-mon[24430]: osdmap e81: 8 total, 8 up, 8 in 2024-05-15T15:31:47.853 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:47 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2803988765' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2314145657"}]: dispatch 2024-05-15T15:31:47.853 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:47 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2314145657"}]: dispatch 2024-05-15T15:31:47.854 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:47 smithi043 ceph-mon[29127]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 511 B/s wr, 130 op/s 2024-05-15T15:31:47.854 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:47 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3688153742"}]': finished 2024-05-15T15:31:47.854 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:47 smithi043 ceph-mon[29127]: osdmap e81: 8 total, 8 up, 8 in 2024-05-15T15:31:47.854 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:47 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2803988765' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2314145657"}]: dispatch 2024-05-15T15:31:47.854 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:47 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2314145657"}]: dispatch 2024-05-15T15:31:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2314145657"}]': finished 2024-05-15T15:31:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: osdmap e82: 8 total, 8 up, 8 in 2024-05-15T15:31:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3527604830' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4208721262"}]: dispatch 2024-05-15T15:31:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:31:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:31:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:31:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:48 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2314145657"}]': finished 2024-05-15T15:31:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: osdmap e82: 8 total, 8 up, 8 in 2024-05-15T15:31:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3527604830' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4208721262"}]: dispatch 2024-05-15T15:31:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:31:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2314145657"}]': finished 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: osdmap e82: 8 total, 8 up, 8 in 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3527604830' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4208721262"}]: dispatch 2024-05-15T15:31:49.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:31:49.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:48 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:49 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:31:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:49 smithi053 ceph-mon[25205]: Adding iSCSI gateway http://:@172.21.15.43:5000 to Dashboard 2024-05-15T15:31:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:49 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:31:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:49 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:31:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:49 smithi053 ceph-mon[25205]: Checking dashboard <-> RGW credentials 2024-05-15T15:31:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:49 smithi053 ceph-mon[25205]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s 2024-05-15T15:31:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:49 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3527604830' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4208721262"}]': finished 2024-05-15T15:31:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:49 smithi053 ceph-mon[25205]: osdmap e83: 8 total, 8 up, 8 in 2024-05-15T15:31:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:49 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/747132330' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3975126779"}]: dispatch 2024-05-15T15:31:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:49 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:31:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[29127]: Adding iSCSI gateway http://:@172.21.15.43:5000 to Dashboard 2024-05-15T15:31:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:31:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:31:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[29127]: Checking dashboard <-> RGW credentials 2024-05-15T15:31:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[29127]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s 2024-05-15T15:31:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3527604830' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4208721262"}]': finished 2024-05-15T15:31:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[29127]: osdmap e83: 8 total, 8 up, 8 in 2024-05-15T15:31:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/747132330' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3975126779"}]: dispatch 2024-05-15T15:31:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:31:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[24430]: Adding iSCSI gateway http://:@172.21.15.43:5000 to Dashboard 2024-05-15T15:31:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:31:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:31:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[24430]: Checking dashboard <-> RGW credentials 2024-05-15T15:31:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[24430]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s 2024-05-15T15:31:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3527604830' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4208721262"}]': finished 2024-05-15T15:31:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[24430]: osdmap e83: 8 total, 8 up, 8 in 2024-05-15T15:31:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/747132330' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3975126779"}]: dispatch 2024-05-15T15:31:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:49 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:31:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:50 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/747132330' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3975126779"}]': finished 2024-05-15T15:31:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:50 smithi053 ceph-mon[25205]: osdmap e84: 8 total, 8 up, 8 in 2024-05-15T15:31:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:50 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/408377044' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/970314706"}]: dispatch 2024-05-15T15:31:51.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:50 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/747132330' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3975126779"}]': finished 2024-05-15T15:31:51.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:50 smithi043 ceph-mon[24430]: osdmap e84: 8 total, 8 up, 8 in 2024-05-15T15:31:51.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:50 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/408377044' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/970314706"}]: dispatch 2024-05-15T15:31:51.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:50 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/747132330' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3975126779"}]': finished 2024-05-15T15:31:51.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:50 smithi043 ceph-mon[29127]: osdmap e84: 8 total, 8 up, 8 in 2024-05-15T15:31:51.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:50 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/408377044' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/970314706"}]: dispatch 2024-05-15T15:31:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:51 smithi053 ceph-mon[25205]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-05-15T15:31:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:51 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/408377044' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/970314706"}]': finished 2024-05-15T15:31:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:51 smithi053 ceph-mon[25205]: osdmap e85: 8 total, 8 up, 8 in 2024-05-15T15:31:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:51 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3221271072' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1710373736"}]: dispatch 2024-05-15T15:31:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:51 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1710373736"}]: dispatch 2024-05-15T15:31:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:51 smithi043 ceph-mon[24430]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-05-15T15:31:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:51 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/408377044' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/970314706"}]': finished 2024-05-15T15:31:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:51 smithi043 ceph-mon[24430]: osdmap e85: 8 total, 8 up, 8 in 2024-05-15T15:31:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:51 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3221271072' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1710373736"}]: dispatch 2024-05-15T15:31:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:51 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1710373736"}]: dispatch 2024-05-15T15:31:52.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:51 smithi043 ceph-mon[29127]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-05-15T15:31:52.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:51 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/408377044' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/970314706"}]': finished 2024-05-15T15:31:52.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:51 smithi043 ceph-mon[29127]: osdmap e85: 8 total, 8 up, 8 in 2024-05-15T15:31:52.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:51 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3221271072' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1710373736"}]: dispatch 2024-05-15T15:31:52.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:51 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1710373736"}]: dispatch 2024-05-15T15:31:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:52 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1710373736"}]': finished 2024-05-15T15:31:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:52 smithi053 ceph-mon[25205]: osdmap e86: 8 total, 8 up, 8 in 2024-05-15T15:31:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:52 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/302228109' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3368735601"}]: dispatch 2024-05-15T15:31:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:52 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3368735601"}]: dispatch 2024-05-15T15:31:53.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:52 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1710373736"}]': finished 2024-05-15T15:31:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:52 smithi043 ceph-mon[24430]: osdmap e86: 8 total, 8 up, 8 in 2024-05-15T15:31:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:52 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/302228109' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3368735601"}]: dispatch 2024-05-15T15:31:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:52 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3368735601"}]: dispatch 2024-05-15T15:31:53.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:31:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:52] "GET /metrics HTTP/1.1" 200 207989 "" "Prometheus/2.33.4" 2024-05-15T15:31:53.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:52 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1710373736"}]': finished 2024-05-15T15:31:53.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:52 smithi043 ceph-mon[29127]: osdmap e86: 8 total, 8 up, 8 in 2024-05-15T15:31:53.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:52 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/302228109' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3368735601"}]: dispatch 2024-05-15T15:31:53.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:52 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3368735601"}]: dispatch 2024-05-15T15:31:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:53 smithi053 ceph-mon[25205]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-05-15T15:31:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:53 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3368735601"}]': finished 2024-05-15T15:31:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:53 smithi053 ceph-mon[25205]: osdmap e87: 8 total, 8 up, 8 in 2024-05-15T15:31:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:53 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2810706140' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/573610528"}]: dispatch 2024-05-15T15:31:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:53 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/573610528"}]: dispatch 2024-05-15T15:31:54.085 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:53 smithi043 ceph-mon[29127]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-05-15T15:31:54.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:53 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3368735601"}]': finished 2024-05-15T15:31:54.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:53 smithi043 ceph-mon[29127]: osdmap e87: 8 total, 8 up, 8 in 2024-05-15T15:31:54.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:53 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2810706140' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/573610528"}]: dispatch 2024-05-15T15:31:54.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:53 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/573610528"}]: dispatch 2024-05-15T15:31:54.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:53 smithi043 ceph-mon[24430]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-05-15T15:31:54.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:53 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3368735601"}]': finished 2024-05-15T15:31:54.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:53 smithi043 ceph-mon[24430]: osdmap e87: 8 total, 8 up, 8 in 2024-05-15T15:31:54.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:53 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2810706140' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/573610528"}]: dispatch 2024-05-15T15:31:54.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:53 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/573610528"}]: dispatch 2024-05-15T15:31:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:54 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/573610528"}]': finished 2024-05-15T15:31:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:54 smithi053 ceph-mon[25205]: osdmap e88: 8 total, 8 up, 8 in 2024-05-15T15:31:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:54 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1262293482' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2411931462"}]: dispatch 2024-05-15T15:31:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:54 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2411931462"}]: dispatch 2024-05-15T15:31:55.085 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:54 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/573610528"}]': finished 2024-05-15T15:31:55.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:54 smithi043 ceph-mon[29127]: osdmap e88: 8 total, 8 up, 8 in 2024-05-15T15:31:55.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:54 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1262293482' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2411931462"}]: dispatch 2024-05-15T15:31:55.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:54 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2411931462"}]: dispatch 2024-05-15T15:31:55.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:54 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/573610528"}]': finished 2024-05-15T15:31:55.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:54 smithi043 ceph-mon[24430]: osdmap e88: 8 total, 8 up, 8 in 2024-05-15T15:31:55.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:54 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1262293482' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2411931462"}]: dispatch 2024-05-15T15:31:55.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:54 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2411931462"}]: dispatch 2024-05-15T15:31:56.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:55 smithi043 ceph-mon[24430]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:56.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:55 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2411931462"}]': finished 2024-05-15T15:31:56.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:55 smithi043 ceph-mon[24430]: osdmap e89: 8 total, 8 up, 8 in 2024-05-15T15:31:56.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:55 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:31:56.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:55 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1099164876' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2411931462"}]: dispatch 2024-05-15T15:31:56.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:55 smithi043 ceph-mon[29127]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:56.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:55 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2411931462"}]': finished 2024-05-15T15:31:56.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:55 smithi043 ceph-mon[29127]: osdmap e89: 8 total, 8 up, 8 in 2024-05-15T15:31:56.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:55 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:31:56.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:55 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1099164876' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2411931462"}]: dispatch 2024-05-15T15:31:56.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:55 smithi053 ceph-mon[25205]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:31:56.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:55 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2411931462"}]': finished 2024-05-15T15:31:56.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:55 smithi053 ceph-mon[25205]: osdmap e89: 8 total, 8 up, 8 in 2024-05-15T15:31:56.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:55 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:31:56.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:55 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1099164876' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2411931462"}]: dispatch 2024-05-15T15:31:56.979 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:56 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1099164876' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2411931462"}]': finished 2024-05-15T15:31:56.979 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:56 smithi043 ceph-mon[24430]: osdmap e90: 8 total, 8 up, 8 in 2024-05-15T15:31:56.979 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:56 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2911938290' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/653624395"}]: dispatch 2024-05-15T15:31:56.980 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:56 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1099164876' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2411931462"}]': finished 2024-05-15T15:31:56.980 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:56 smithi043 ceph-mon[29127]: osdmap e90: 8 total, 8 up, 8 in 2024-05-15T15:31:56.980 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:56 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2911938290' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/653624395"}]: dispatch 2024-05-15T15:31:57.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:31:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:31:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:31:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:56 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1099164876' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2411931462"}]': finished 2024-05-15T15:31:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:56 smithi053 ceph-mon[25205]: osdmap e90: 8 total, 8 up, 8 in 2024-05-15T15:31:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:56 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2911938290' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/653624395"}]: dispatch 2024-05-15T15:31:58.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:57 smithi043 ceph-mon[29127]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:31:58.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:57 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2911938290' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/653624395"}]': finished 2024-05-15T15:31:58.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:57 smithi043 ceph-mon[29127]: osdmap e91: 8 total, 8 up, 8 in 2024-05-15T15:31:58.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:57 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/471293128' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1103755388"}]: dispatch 2024-05-15T15:31:58.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:57 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1103755388"}]: dispatch 2024-05-15T15:31:58.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:57 smithi043 ceph-mon[24430]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:31:58.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:57 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2911938290' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/653624395"}]': finished 2024-05-15T15:31:58.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:57 smithi043 ceph-mon[24430]: osdmap e91: 8 total, 8 up, 8 in 2024-05-15T15:31:58.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:57 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/471293128' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1103755388"}]: dispatch 2024-05-15T15:31:58.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:57 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1103755388"}]: dispatch 2024-05-15T15:31:58.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:57 smithi053 ceph-mon[25205]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:31:58.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:57 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2911938290' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/653624395"}]': finished 2024-05-15T15:31:58.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:57 smithi053 ceph-mon[25205]: osdmap e91: 8 total, 8 up, 8 in 2024-05-15T15:31:58.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:57 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/471293128' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1103755388"}]: dispatch 2024-05-15T15:31:58.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:57 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1103755388"}]: dispatch 2024-05-15T15:31:59.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:58 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1103755388"}]': finished 2024-05-15T15:31:59.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:58 smithi043 ceph-mon[24430]: osdmap e92: 8 total, 8 up, 8 in 2024-05-15T15:31:59.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:58 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2254876603' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/663705201"}]: dispatch 2024-05-15T15:31:59.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:58 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1103755388"}]': finished 2024-05-15T15:31:59.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:58 smithi043 ceph-mon[29127]: osdmap e92: 8 total, 8 up, 8 in 2024-05-15T15:31:59.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:58 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2254876603' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/663705201"}]: dispatch 2024-05-15T15:31:59.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:58 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1103755388"}]': finished 2024-05-15T15:31:59.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:58 smithi053 ceph-mon[25205]: osdmap e92: 8 total, 8 up, 8 in 2024-05-15T15:31:59.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:58 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2254876603' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/663705201"}]: dispatch 2024-05-15T15:32:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:59 smithi043 ceph-mon[24430]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:59 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2254876603' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/663705201"}]': finished 2024-05-15T15:32:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:59 smithi043 ceph-mon[24430]: osdmap e93: 8 total, 8 up, 8 in 2024-05-15T15:32:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:31:59 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1946640855' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/970314706"}]: dispatch 2024-05-15T15:32:00.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:59 smithi043 ceph-mon[29127]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:00.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:59 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2254876603' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/663705201"}]': finished 2024-05-15T15:32:00.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:59 smithi043 ceph-mon[29127]: osdmap e93: 8 total, 8 up, 8 in 2024-05-15T15:32:00.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:31:59 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1946640855' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/970314706"}]: dispatch 2024-05-15T15:32:00.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:59 smithi053 ceph-mon[25205]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:00.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:59 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2254876603' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/663705201"}]': finished 2024-05-15T15:32:00.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:59 smithi053 ceph-mon[25205]: osdmap e93: 8 total, 8 up, 8 in 2024-05-15T15:32:00.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:31:59 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1946640855' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/970314706"}]: dispatch 2024-05-15T15:32:01.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:00 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1946640855' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/970314706"}]': finished 2024-05-15T15:32:01.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:00 smithi043 ceph-mon[24430]: osdmap e94: 8 total, 8 up, 8 in 2024-05-15T15:32:01.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:00 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1208920169' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3154897789"}]: dispatch 2024-05-15T15:32:01.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:00 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3154897789"}]: dispatch 2024-05-15T15:32:01.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:00 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1946640855' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/970314706"}]': finished 2024-05-15T15:32:01.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:00 smithi043 ceph-mon[29127]: osdmap e94: 8 total, 8 up, 8 in 2024-05-15T15:32:01.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:00 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1208920169' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3154897789"}]: dispatch 2024-05-15T15:32:01.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:00 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3154897789"}]: dispatch 2024-05-15T15:32:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:00 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1946640855' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/970314706"}]': finished 2024-05-15T15:32:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:00 smithi053 ceph-mon[25205]: osdmap e94: 8 total, 8 up, 8 in 2024-05-15T15:32:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:00 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1208920169' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3154897789"}]: dispatch 2024-05-15T15:32:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:00 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3154897789"}]: dispatch 2024-05-15T15:32:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:01 smithi043 ceph-mon[24430]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:01 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3154897789"}]': finished 2024-05-15T15:32:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:01 smithi043 ceph-mon[24430]: osdmap e95: 8 total, 8 up, 8 in 2024-05-15T15:32:02.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:01 smithi043 ceph-mon[29127]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:02.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:01 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3154897789"}]': finished 2024-05-15T15:32:02.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:01 smithi043 ceph-mon[29127]: osdmap e95: 8 total, 8 up, 8 in 2024-05-15T15:32:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:01 smithi053 ceph-mon[25205]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:01 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3154897789"}]': finished 2024-05-15T15:32:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:01 smithi053 ceph-mon[25205]: osdmap e95: 8 total, 8 up, 8 in 2024-05-15T15:32:03.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:32:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:02] "GET /metrics HTTP/1.1" 200 208086 "" "Prometheus/2.33.4" 2024-05-15T15:32:04.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:03 smithi043 ceph-mon[24430]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:32:04.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:03 smithi043 ceph-mon[29127]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:32:04.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:03 smithi053 ceph-mon[25205]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:32:05.932 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:05 smithi053 ceph-mon[25205]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:05.932 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:05 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:06.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:05 smithi043 ceph-mon[24430]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:06.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:05 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:06.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:05 smithi043 ceph-mon[29127]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:06.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:05 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:07.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:32:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:32:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:07 smithi043 ceph-mon[24430]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:32:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:07 smithi043 ceph-mon[29127]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:32:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:07 smithi053 ceph-mon[25205]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:32:10.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:09 smithi043 ceph-mon[24430]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:09 smithi043 ceph-mon[29127]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:09 smithi053 ceph-mon[25205]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:11 smithi043 ceph-mon[24430]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:32:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:11 smithi043 ceph-mon[29127]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:32:11.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:11 smithi053 ceph-mon[25205]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:32:13.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:32:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:12] "GET /metrics HTTP/1.1" 200 208086 "" "Prometheus/2.33.4" 2024-05-15T15:32:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:13 smithi053 ceph-mon[25205]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-05-15T15:32:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:13 smithi043 ceph-mon[24430]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-05-15T15:32:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:13 smithi043 ceph-mon[29127]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-05-15T15:32:15.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:15 smithi053 ceph-mon[25205]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:15.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:15 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:15 smithi043 ceph-mon[24430]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:15 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:15 smithi043 ceph-mon[29127]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:15 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:17.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:32:16 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:32:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:17 smithi053 ceph-mon[25205]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:17 smithi043 ceph-mon[29127]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:17 smithi043 ceph-mon[24430]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:19 smithi053 ceph-mon[25205]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:19.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:19 smithi043 ceph-mon[24430]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:19 smithi043 ceph-mon[29127]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:21 smithi043 ceph-mon[29127]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:21.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:21 smithi043 ceph-mon[24430]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:21 smithi053 ceph-mon[25205]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:23.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:32:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:22] "GET /metrics HTTP/1.1" 200 208049 "" "Prometheus/2.33.4" 2024-05-15T15:32:23.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:23 smithi053 ceph-mon[25205]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:23.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:23 smithi043 ceph-mon[24430]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:23 smithi043 ceph-mon[29127]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:25 smithi053 ceph-mon[25205]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:25 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:25 smithi043 ceph-mon[24430]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:25 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:25 smithi043 ceph-mon[29127]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:25 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:27.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:32:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:32:27.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:27 smithi043 ceph-mon[24430]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:27.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:27 smithi043 ceph-mon[29127]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:27 smithi053 ceph-mon[25205]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:29.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:29 smithi053 ceph-mon[25205]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:29.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:29 smithi043 ceph-mon[24430]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:29 smithi043 ceph-mon[29127]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:31.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:31 smithi043 ceph-mon[24430]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:31.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:31 smithi043 ceph-mon[29127]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:31.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:31 smithi053 ceph-mon[25205]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:33.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:32:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:32] "GET /metrics HTTP/1.1" 200 208009 "" "Prometheus/2.33.4" 2024-05-15T15:32:33.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:33 smithi053 ceph-mon[25205]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:33.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:33 smithi043 ceph-mon[29127]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:33.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:33 smithi043 ceph-mon[24430]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:35.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:35 smithi053 ceph-mon[25205]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:35.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:35 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:35.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:35 smithi043 ceph-mon[24430]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:35 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:35 smithi043 ceph-mon[29127]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:35 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:37.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:32:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:32:37.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:37 smithi053 ceph-mon[25205]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:37 smithi043 ceph-mon[29127]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:37.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:37 smithi043 ceph-mon[24430]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:39 smithi053 ceph-mon[25205]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:39.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:39 smithi043 ceph-mon[24430]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:39 smithi043 ceph-mon[29127]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:41.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:41 smithi043 ceph-mon[24430]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:41.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:41 smithi043 ceph-mon[29127]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:41 smithi053 ceph-mon[25205]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:43.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:32:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:42] "GET /metrics HTTP/1.1" 200 208009 "" "Prometheus/2.33.4" 2024-05-15T15:32:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:43 smithi053 ceph-mon[25205]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:43 smithi043 ceph-mon[24430]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:43 smithi043 ceph-mon[29127]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:32:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:32:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:32:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:32:44.739 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:32:44.740 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:32:44.740 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:32:44.740 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:32:44.740 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:32:44.740 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:32:44.740 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:32:44.740 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:32:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:45 smithi053 ceph-mon[25205]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:45 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:45.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:45 smithi043 ceph-mon[24430]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:45.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:45 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:45 smithi043 ceph-mon[29127]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:45 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:47.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:32:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:32:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:47 smithi053 ceph-mon[25205]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:47.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:47 smithi043 ceph-mon[24430]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:47 smithi043 ceph-mon[29127]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:49 smithi053 ceph-mon[25205]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:49 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:32:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:49 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:32:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:49 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:32:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:49 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:32:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:49 smithi043 ceph-mon[24430]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:49 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:32:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:49 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:32:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:49 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:32:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:49 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:32:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:49 smithi043 ceph-mon[29127]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:49 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:32:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:49 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:32:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:49 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:32:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:49 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:32:51.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:51 smithi043 ceph-mon[24430]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:51.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:51 smithi043 ceph-mon[29127]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:51 smithi053 ceph-mon[25205]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:53.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:32:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:52] "GET /metrics HTTP/1.1" 200 208000 "" "Prometheus/2.33.4" 2024-05-15T15:32:53.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:53 smithi053 ceph-mon[25205]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:53 smithi043 ceph-mon[24430]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:53 smithi043 ceph-mon[29127]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:55.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:55 smithi053 ceph-mon[25205]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:55 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:55.786 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:55 smithi043 ceph-mon[24430]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:55.786 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:55 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:55.786 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:55 smithi043 ceph-mon[29127]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:55.786 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:55 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:32:57.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:32:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:32:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:32:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:57 smithi053 ceph-mon[25205]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:57.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:57 smithi043 ceph-mon[24430]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:57 smithi043 ceph-mon[29127]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:32:59.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:32:59 smithi053 ceph-mon[25205]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:32:59 smithi043 ceph-mon[24430]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:32:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:32:59 smithi043 ceph-mon[29127]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:01 smithi043 ceph-mon[24430]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:01.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:01 smithi043 ceph-mon[29127]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:01 smithi053 ceph-mon[25205]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:03.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:33:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:02] "GET /metrics HTTP/1.1" 200 208004 "" "Prometheus/2.33.4" 2024-05-15T15:33:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:03 smithi053 ceph-mon[25205]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:03.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:03 smithi043 ceph-mon[24430]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:03 smithi043 ceph-mon[29127]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:05 smithi053 ceph-mon[25205]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:05 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:05 smithi043 ceph-mon[24430]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:05 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:05 smithi043 ceph-mon[29127]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:05 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:07.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:33:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:33:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:07 smithi053 ceph-mon[25205]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:07.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:07 smithi043 ceph-mon[24430]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:07.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:07 smithi043 ceph-mon[29127]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:09.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:09 smithi053 ceph-mon[25205]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:09.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:09 smithi043 ceph-mon[24430]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:09 smithi043 ceph-mon[29127]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:11 smithi043 ceph-mon[24430]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:11 smithi043 ceph-mon[29127]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:11.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:11 smithi053 ceph-mon[25205]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:13.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:33:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:12] "GET /metrics HTTP/1.1" 200 208004 "" "Prometheus/2.33.4" 2024-05-15T15:33:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:13 smithi053 ceph-mon[25205]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:13 smithi043 ceph-mon[24430]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:13 smithi043 ceph-mon[29127]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:15.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:15 smithi053 ceph-mon[25205]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:15.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:15 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:15.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:15 smithi043 ceph-mon[29127]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:15 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:15 smithi043 ceph-mon[24430]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:15 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:17.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:33:16 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:33:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:17 smithi053 ceph-mon[25205]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:17.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:17 smithi043 ceph-mon[24430]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:17 smithi043 ceph-mon[29127]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:19 smithi053 ceph-mon[25205]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:19 smithi043 ceph-mon[24430]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:19 smithi043 ceph-mon[29127]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:21.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:21 smithi043 ceph-mon[24430]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:21 smithi043 ceph-mon[29127]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:21 smithi053 ceph-mon[25205]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:23.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:33:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:22] "GET /metrics HTTP/1.1" 200 208001 "" "Prometheus/2.33.4" 2024-05-15T15:33:23.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:23 smithi053 ceph-mon[25205]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:23.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:23 smithi043 ceph-mon[24430]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:23 smithi043 ceph-mon[29127]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:25 smithi053 ceph-mon[25205]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:25 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:25.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:25 smithi043 ceph-mon[29127]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:25 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:25 smithi043 ceph-mon[24430]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:25 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:27.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:33:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:33:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:27 smithi053 ceph-mon[25205]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:27.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:27 smithi043 ceph-mon[24430]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:27.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:27 smithi043 ceph-mon[29127]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:29.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:29 smithi053 ceph-mon[25205]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:29.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:29 smithi043 ceph-mon[24430]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:29 smithi043 ceph-mon[29127]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:31.585 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:31 smithi043 ceph-mon[29127]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:31.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:31 smithi043 ceph-mon[24430]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:31.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:31 smithi053 ceph-mon[25205]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:33.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:33:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:32] "GET /metrics HTTP/1.1" 200 208003 "" "Prometheus/2.33.4" 2024-05-15T15:33:33.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:33 smithi053 ceph-mon[25205]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:33.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:33 smithi043 ceph-mon[24430]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:33 smithi043 ceph-mon[29127]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:35.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:35 smithi053 ceph-mon[25205]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:35.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:35 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:35 smithi043 ceph-mon[24430]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:35 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:35 smithi043 ceph-mon[29127]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:35 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:37.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:33:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:33:37.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:37 smithi053 ceph-mon[25205]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:37.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:37 smithi043 ceph-mon[24430]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:37 smithi043 ceph-mon[29127]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:39 smithi053 ceph-mon[25205]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:39.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:39 smithi043 ceph-mon[24430]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:39 smithi043 ceph-mon[29127]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:41.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:41 smithi043 ceph-mon[24430]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:41.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:41 smithi043 ceph-mon[29127]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:41.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:41 smithi053 ceph-mon[25205]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:43.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:33:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:42] "GET /metrics HTTP/1.1" 200 208003 "" "Prometheus/2.33.4" 2024-05-15T15:33:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:43 smithi053 ceph-mon[25205]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:43.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:43 smithi043 ceph-mon[24430]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:43 smithi043 ceph-mon[29127]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:33:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:33:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:33:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:33:44.790 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:33:44.790 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:33:44.790 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:33:44.790 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:33:44.790 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:33:44.791 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:33:44.791 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:33:44.791 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:33:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:45 smithi053 ceph-mon[25205]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:45 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:45.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:45 smithi043 ceph-mon[24430]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:45 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:45 smithi043 ceph-mon[29127]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:45 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:47.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:33:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:33:47.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:47 smithi043 ceph-mon[24430]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:47 smithi043 ceph-mon[29127]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:47.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:47 smithi053 ceph-mon[25205]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:49 smithi043 ceph-mon[24430]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:33:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:49 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:33:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:49 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:33:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:49 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:33:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:49 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:33:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:49 smithi043 ceph-mon[29127]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:33:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:49 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:33:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:49 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:33:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:49 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:33:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:49 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:33:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:49 smithi053 ceph-mon[25205]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:33:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:49 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:33:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:49 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:33:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:49 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:33:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:49 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:33:51.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:51 smithi043 ceph-mon[24430]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:51.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:51 smithi043 ceph-mon[29127]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:51 smithi053 ceph-mon[25205]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:53.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:33:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:52] "GET /metrics HTTP/1.1" 200 207993 "" "Prometheus/2.33.4" 2024-05-15T15:33:53.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:53 smithi053 ceph-mon[25205]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:33:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:53 smithi043 ceph-mon[24430]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:33:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:53 smithi043 ceph-mon[29127]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:33:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:55 smithi043 ceph-mon[24430]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:33:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:55 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:55.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:55 smithi043 ceph-mon[29127]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:33:55.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:55 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:55 smithi053 ceph-mon[25205]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:33:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:55 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:33:57.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:33:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:33:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:33:57.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:57 smithi043 ceph-mon[24430]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:57 smithi043 ceph-mon[29127]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:57 smithi053 ceph-mon[25205]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:33:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:33:59 smithi043 ceph-mon[29127]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:33:59 smithi043 ceph-mon[24430]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:33:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:33:59 smithi053 ceph-mon[25205]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:01 smithi043 ceph-mon[24430]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:01.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:01 smithi043 ceph-mon[29127]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:01 smithi053 ceph-mon[25205]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:03.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:34:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:02] "GET /metrics HTTP/1.1" 200 207979 "" "Prometheus/2.33.4" 2024-05-15T15:34:03.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:03 smithi043 ceph-mon[24430]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:03 smithi043 ceph-mon[29127]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:03 smithi053 ceph-mon[25205]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:05.682 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:05 smithi053 ceph-mon[25205]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:05.682 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:05 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:05.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:05 smithi043 ceph-mon[24430]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:05 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:05 smithi043 ceph-mon[29127]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:05 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:07.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:34:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:34:07.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:07 smithi043 ceph-mon[29127]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:07.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:07 smithi043 ceph-mon[24430]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:07 smithi053 ceph-mon[25205]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:09 smithi043 ceph-mon[24430]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:09 smithi043 ceph-mon[29127]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:09.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:09 smithi053 ceph-mon[25205]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:11.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:11 smithi043 ceph-mon[24430]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:11 smithi043 ceph-mon[29127]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:11 smithi053 ceph-mon[25205]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:13.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:34:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:12] "GET /metrics HTTP/1.1" 200 207979 "" "Prometheus/2.33.4" 2024-05-15T15:34:13.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:13 smithi043 ceph-mon[24430]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:13.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:13 smithi043 ceph-mon[29127]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:13 smithi053 ceph-mon[25205]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:15.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:15 smithi043 ceph-mon[24430]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:15 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:15 smithi043 ceph-mon[29127]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:15 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:15 smithi053 ceph-mon[25205]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:15 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:17.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:34:16 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:34:17.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:17 smithi043 ceph-mon[24430]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:17 smithi043 ceph-mon[29127]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:17 smithi053 ceph-mon[25205]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:19 smithi043 ceph-mon[24430]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:19 smithi043 ceph-mon[29127]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:19.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:19 smithi053 ceph-mon[25205]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:21.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:21 smithi043 ceph-mon[24430]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:21 smithi043 ceph-mon[29127]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:21 smithi053 ceph-mon[25205]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:23.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:34:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:22] "GET /metrics HTTP/1.1" 200 207946 "" "Prometheus/2.33.4" 2024-05-15T15:34:23.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:23 smithi043 ceph-mon[24430]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:23 smithi043 ceph-mon[29127]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:23.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:23 smithi053 ceph-mon[25205]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:25 smithi043 ceph-mon[24430]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:25 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:25 smithi043 ceph-mon[29127]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:25 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:25 smithi053 ceph-mon[25205]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:25 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:27.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:34:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:34:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:27 smithi043 ceph-mon[24430]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:27.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:27 smithi043 ceph-mon[29127]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:27 smithi053 ceph-mon[25205]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:29.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:29 smithi043 ceph-mon[24430]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:29 smithi043 ceph-mon[29127]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:29 smithi053 ceph-mon[25205]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:31.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:31 smithi043 ceph-mon[24430]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:31.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:31 smithi043 ceph-mon[29127]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:31.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:31 smithi053 ceph-mon[25205]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:33.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:34:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:32] "GET /metrics HTTP/1.1" 200 207943 "" "Prometheus/2.33.4" 2024-05-15T15:34:33.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:33 smithi043 ceph-mon[24430]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:33 smithi043 ceph-mon[29127]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:33.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:33 smithi053 ceph-mon[25205]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:35.561 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:35 smithi043 ceph-mon[24430]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:35.561 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:35 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:35.561 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:35 smithi043 ceph-mon[29127]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:35.561 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:35 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:35.682 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:35 smithi053 ceph-mon[25205]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:35.682 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:35 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:36.047 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-05-15T15:34:37.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:34:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:34:37.410 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:37 smithi043 ceph-mon[24430]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:37.410 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:37 smithi043 ceph-mon[29127]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:37 smithi053 ceph-mon[25205]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:38.501 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-05-15T15:34:39.678 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:39 smithi043 ceph-mon[24430]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:39.679 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:39 smithi043 ceph-mon[29127]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:39 smithi053 ceph-mon[25205]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:40.878 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph config set global log_to_journald false --force' 2024-05-15T15:34:41.446 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:41 smithi043 ceph-mon[24430]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:41.447 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:41 smithi043 ceph-mon[29127]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:41 smithi053 ceph-mon[25205]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:42.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:34:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:42] "GET /metrics HTTP/1.1" 200 207943 "" "Prometheus/2.33.4" 2024-05-15T15:34:43.163 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T15:34:43.480 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:43 smithi043 ceph-mon[24430]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:43.481 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:43 smithi043 ceph-mon[29127]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:43 smithi053 ceph-mon[25205]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:44.651 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:34:44.651 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:34:44.651 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:34:44.651 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:34:44.651 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:34:44.652 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:34:44.652 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:34:44.652 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:34:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:34:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:34:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:34:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (3m) 2m ago 3m 17.5M - ba2b418f427c c7485334fa8c 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (3m) 2m ago 3m 49.0M - 8.3.5 dad864ee21e9 3e2dc1d42275 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 running (3m) 2m ago 3m 63.8M - 3.5 e1d6a67b021e 2a6c547e80fa 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443 running (8m) 2m ago 8m 410M - 17.2.0 e1d6a67b021e da9fa0ee69bf 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:9283 running (10m) 2m ago 10m 454M - 17.2.0 e1d6a67b021e faaa138d25ee 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (10m) 2m ago 10m 44.9M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (8m) 2m ago 8m 40.9M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (8m) 2m ago 8m 40.7M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (4m) 2m ago 4m 14.0M - 1dbe0e931976 8cfbd4656c29 2024-05-15T15:34:45.089 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (4m) 2m ago 4m 16.9M - 1dbe0e931976 c6fac2ae2513 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (8m) 2m ago 8m 46.8M 4017M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (7m) 2m ago 7m 50.9M 4017M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (7m) 2m ago 7m 43.7M 4017M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (6m) 2m ago 6m 45.2M 4017M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (6m) 2m ago 6m 44.8M 4273M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (6m) 2m ago 6m 42.6M 4273M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (5m) 2m ago 5m 45.7M 4273M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (5m) 2m ago 5m 47.4M 4273M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (3m) 2m ago 4m 44.1M - 514e6a882f6e 8e95fb39ec80 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (3m) 2m ago 3m 69.1M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:34:45.090 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (3m) 2m ago 3m 67.7M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:34:45.654 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:45 smithi043 ceph-mon[29127]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:45.654 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:45 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:45.655 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:45 smithi043 ceph-mon[24430]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:45.655 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:45 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:45.679 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions' 2024-05-15T15:34:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:45 smithi053 ceph-mon[25205]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:45 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:46.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:46 smithi043 ceph-mon[24430]: from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:34:46.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:46 smithi043 ceph-mon[29127]: from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:34:46.840 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:46 smithi053 ceph-mon[25205]: from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:34:47.129 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:34:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:34:47.585 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:34:47.585 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:34:47.586 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-05-15T15:34:47.587 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:34:47.587 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:34:47.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:47 smithi043 ceph-mon[24430]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:47.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:47 smithi043 ceph-mon[29127]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:47 smithi053 ceph-mon[25205]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:48.157 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph -s' 2024-05-15T15:34:48.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:48 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2249822100' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:34:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:48 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2249822100' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:34:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:48 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2249822100' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:34:49.675 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:49 smithi043 ceph-mon[29127]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:49.675 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:49 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:34:49.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:49 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:34:49.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:49 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:34:49.676 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:49 smithi043 ceph-mon[24430]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:49.676 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:49 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:34:49.676 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:49 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:34:49.676 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:49 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:34:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:49 smithi053 ceph-mon[25205]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:49 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:34:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:49 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:34:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:49 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:34:50.139 INFO:teuthology.orchestra.run.smithi043.stdout: cluster: 2024-05-15T15:34:50.139 INFO:teuthology.orchestra.run.smithi043.stdout: id: 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:34:50.139 INFO:teuthology.orchestra.run.smithi043.stdout: health: HEALTH_OK 2024-05-15T15:34:50.139 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:34:50.139 INFO:teuthology.orchestra.run.smithi043.stdout: services: 2024-05-15T15:34:50.139 INFO:teuthology.orchestra.run.smithi043.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-05-15T15:34:50.139 INFO:teuthology.orchestra.run.smithi043.stdout: mgr: y(active, since 5m), standbys: x 2024-05-15T15:34:50.139 INFO:teuthology.orchestra.run.smithi043.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-05-15T15:34:50.139 INFO:teuthology.orchestra.run.smithi043.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-15T15:34:50.139 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:34:50.140 INFO:teuthology.orchestra.run.smithi043.stdout: data: 2024-05-15T15:34:50.140 INFO:teuthology.orchestra.run.smithi043.stdout: pools: 6 pools, 161 pgs 2024-05-15T15:34:50.140 INFO:teuthology.orchestra.run.smithi043.stdout: objects: 209 objects, 585 KiB 2024-05-15T15:34:50.140 INFO:teuthology.orchestra.run.smithi043.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:34:50.140 INFO:teuthology.orchestra.run.smithi043.stdout: pgs: 161 active+clean 2024-05-15T15:34:50.140 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:34:50.140 INFO:teuthology.orchestra.run.smithi043.stdout: io: 2024-05-15T15:34:50.140 INFO:teuthology.orchestra.run.smithi043.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-15T15:34:50.140 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:34:50.738 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ls' 2024-05-15T15:34:50.762 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:50 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:34:50.762 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:50 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/469646210' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:34:50.762 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:50 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:34:50.762 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:50 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/469646210' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:34:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:50 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:34:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:50 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/469646210' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:34:51.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:51 smithi043 ceph-mon[24430]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:51.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:51 smithi043 ceph-mon[29127]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:51 smithi053 ceph-mon[25205]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:52.545 INFO:teuthology.orchestra.run.smithi043.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-15T15:34:52.546 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi043=a;count:1 2024-05-15T15:34:52.546 INFO:teuthology.orchestra.run.smithi043.stdout:grafana ?:3000 1/1 3m ago 5m smithi053=a;count:1 2024-05-15T15:34:52.546 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-05-15T15:34:52.546 INFO:teuthology.orchestra.run.smithi043.stdout:mgr 2/2 3m ago 8m smithi043=y;smithi053=x;count:2 2024-05-15T15:34:52.546 INFO:teuthology.orchestra.run.smithi043.stdout:mon 3/3 3m ago 9m smithi043:172.21.15.43=a;smithi043:[v2:172.21.15.43:3301,v1:172.21.15.43:6790]=c;smithi053:172.21.15.53=b;count:3 2024-05-15T15:34:52.546 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi043=a;smithi053=b;count:2 2024-05-15T15:34:52.546 INFO:teuthology.orchestra.run.smithi043.stdout:osd 8 3m ago - 2024-05-15T15:34:52.546 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus ?:9095 1/1 3m ago 5m smithi053=a;count:1 2024-05-15T15:34:52.546 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-05-15T15:34:52.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:34:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:52] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-05-15T15:34:53.200 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-15T15:34:53.762 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:53 smithi043 ceph-mon[24430]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:53.762 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:53 smithi043 ceph-mon[24430]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:34:53.763 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:53 smithi043 ceph-mon[29127]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:53.763 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:53 smithi043 ceph-mon[29127]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:34:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:53 smithi053 ceph-mon[25205]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:53 smithi053 ceph-mon[25205]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:34:55.657 INFO:teuthology.orchestra.run.smithi043.stdout:Scheduled to redeploy mgr.x on host 'smithi053' 2024-05-15T15:34:55.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:55 smithi043 ceph-mon[24430]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:55 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:55 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1757712670' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-15T15:34:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:55 smithi043 ceph-mon[29127]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:55 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:55 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1757712670' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-15T15:34:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:55 smithi053 ceph-mon[25205]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:55 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:34:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:55 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1757712670' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-15T15:34:56.514 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps --refresh' 2024-05-15T15:34:56.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:56 smithi043 ceph-mon[29127]: from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:34:56.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:56 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:34:56.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:56 smithi043 ceph-mon[29127]: Schedule redeploy daemon mgr.x 2024-05-15T15:34:56.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:56 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:34:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:56 smithi043 ceph-mon[24430]: from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:34:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:56 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:34:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:56 smithi043 ceph-mon[24430]: Schedule redeploy daemon mgr.x 2024-05-15T15:34:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:56 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:34:57.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:34:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:34:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:34:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:56 smithi053 ceph-mon[25205]: from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:34:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:56 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:34:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:56 smithi053 ceph-mon[25205]: Schedule redeploy daemon mgr.x 2024-05-15T15:34:57.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:56 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:34:57.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:57 smithi043 ceph-mon[24430]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:57 smithi043 ceph-mon[29127]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:58.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:57 smithi053 ceph-mon[25205]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (3m) 3m ago 4m 17.5M - ba2b418f427c c7485334fa8c 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (3m) 3m ago 3m 49.0M - 8.3.5 dad864ee21e9 3e2dc1d42275 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 running (3m) 3m ago 3m 63.8M - 3.5 e1d6a67b021e 2a6c547e80fa 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e da9fa0ee69bf 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e faaa138d25ee 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (10m) 3m ago 10m 44.9M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (8m) 3m ago 8m 40.9M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (9m) 3m ago 9m 40.7M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (4m) 3m ago 4m 14.0M - 1dbe0e931976 8cfbd4656c29 2024-05-15T15:34:58.567 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (4m) 3m ago 4m 16.9M - 1dbe0e931976 c6fac2ae2513 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (8m) 3m ago 8m 46.8M 4017M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (7m) 3m ago 7m 50.9M 4017M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (7m) 3m ago 7m 43.7M 4017M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (7m) 3m ago 7m 45.2M 4017M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (6m) 3m ago 6m 44.8M 4273M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (6m) 3m ago 6m 42.6M 4273M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (6m) 3m ago 5m 45.7M 4273M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (5m) 3m ago 5m 47.4M 4273M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (3m) 3m ago 4m 44.1M - 514e6a882f6e 8e95fb39ec80 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (3m) 3m ago 3m 69.1M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:34:58.568 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (3m) 3m ago 3m 67.7M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:34:59.248 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'sleep 180' 2024-05-15T15:34:59.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:59 smithi043 ceph-mon[24430]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:59.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:34:59 smithi043 ceph-mon[24430]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:34:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:59 smithi043 ceph-mon[29127]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:34:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:34:59 smithi043 ceph-mon[29127]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:35:00.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:59 smithi053 ceph-mon[25205]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:00.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:34:59 smithi053 ceph-mon[25205]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:35:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:01 smithi043 ceph-mon[24430]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:01.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:01 smithi043 ceph-mon[29127]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:01 smithi053 ceph-mon[25205]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:03.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:35:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:02] "GET /metrics HTTP/1.1" 200 207936 "" "Prometheus/2.33.4" 2024-05-15T15:35:03.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:03 smithi043 ceph-mon[24430]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:03 smithi043 ceph-mon[29127]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:03 smithi053 ceph-mon[25205]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:05.682 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:05 smithi053 ceph-mon[25205]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:05.682 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:05 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:05.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:05 smithi043 ceph-mon[24430]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:05.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:05 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:05 smithi043 ceph-mon[29127]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:05 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:07.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:35:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:35:07.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:07 smithi043 ceph-mon[24430]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:07.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:07 smithi043 ceph-mon[29127]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:07 smithi053 ceph-mon[25205]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:09.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:09 smithi043 ceph-mon[24430]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:09 smithi043 ceph-mon[29127]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:09.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:09 smithi053 ceph-mon[25205]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:11.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:11 smithi043 ceph-mon[24430]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:11.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:11 smithi043 ceph-mon[29127]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:11.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:11 smithi053 ceph-mon[25205]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:13.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:35:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:12] "GET /metrics HTTP/1.1" 200 207936 "" "Prometheus/2.33.4" 2024-05-15T15:35:13.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:13 smithi043 ceph-mon[24430]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:13 smithi043 ceph-mon[29127]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:13.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:13 smithi053 ceph-mon[25205]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:15.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:15 smithi043 ceph-mon[24430]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:15.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:15 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:15.604 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:15 smithi043 ceph-mon[29127]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:15.604 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:15 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:15 smithi053 ceph-mon[25205]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:15 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:17.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:35:16 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:35:17.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:17 smithi043 ceph-mon[24430]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:17 smithi043 ceph-mon[29127]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:17 smithi053 ceph-mon[25205]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:19.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:19 smithi043 ceph-mon[24430]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:19 smithi043 ceph-mon[29127]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:19.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:19 smithi053 ceph-mon[25205]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:21.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:21 smithi043 ceph-mon[24430]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:21 smithi043 ceph-mon[29127]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:21 smithi053 ceph-mon[25205]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:23.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:35:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:22] "GET /metrics HTTP/1.1" 200 207938 "" "Prometheus/2.33.4" 2024-05-15T15:35:23.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:23 smithi043 ceph-mon[24430]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:23 smithi043 ceph-mon[29127]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:23.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:23 smithi053 ceph-mon[25205]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:25.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:25 smithi043 ceph-mon[24430]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:25 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:25 smithi043 ceph-mon[29127]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:25 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:25.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:25 smithi053 ceph-mon[25205]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:25 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:27.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:35:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:35:27.660 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:27 smithi053 ceph-mon[25205]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:27.700 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:27 smithi043 ceph-mon[24430]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:27.701 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:27 smithi043 ceph-mon[29127]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:28.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:28 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:28.845 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:28 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:35:28.845 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:28 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:35:28.845 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:28 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:28.846 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:28 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:28.846 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:28 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:35:28.846 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:28 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:35:28.846 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:28 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:28 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:28 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:35:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:28 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:35:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:28 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:30.129 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:29 smithi053 ceph-mon[25205]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:30.146 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:29 smithi043 ceph-mon[24430]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:30.146 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:29 smithi043 ceph-mon[29127]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:32.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[24430]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:32.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:32.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:32.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:32.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:35:32.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:35:32.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:35:32.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:35:32.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[29127]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:32.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:32.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:32.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:32.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:35:32.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:35:32.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:35:32.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:31 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:35:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:31 smithi053 ceph-mon[25205]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:31 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:31 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:31 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:35:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:31 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:35:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:31 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:35:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:31 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:35:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:31 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:35:32.980 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:35:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:32] "GET /metrics HTTP/1.1" 200 207938 "" "Prometheus/2.33.4" 2024-05-15T15:35:33.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:32 smithi043 ceph-mon[24430]: Deploying daemon mgr.x on smithi053 2024-05-15T15:35:33.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:32 smithi043 ceph-mon[29127]: Deploying daemon mgr.x on smithi053 2024-05-15T15:35:33.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:32 smithi053 ceph-mon[25205]: Deploying daemon mgr.x on smithi053 2024-05-15T15:35:33.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:35:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:35:33.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:35:33.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:35:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:35:33.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:35:34.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:33 smithi043 ceph-mon[24430]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:34.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:33 smithi043 ceph-mon[29127]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:34.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:33 smithi053 ceph-mon[25205]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:36.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:35 smithi043 ceph-mon[24430]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:36.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:35 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:36.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:35 smithi043 ceph-mon[29127]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:36.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:35 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:36.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:35 smithi053 ceph-mon[25205]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:36.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:35 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:37.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:35:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:35:38.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:37 smithi043 ceph-mon[24430]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:38.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:37 smithi043 ceph-mon[29127]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:38.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:37 smithi053 ceph-mon[25205]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:40.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:39 smithi043 ceph-mon[24430]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:40.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:39 smithi043 ceph-mon[29127]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:40.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:39 smithi053 ceph-mon[25205]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:41.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:41 smithi043 ceph-mon[24430]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:41.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:41 smithi043 ceph-mon[29127]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:41.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:41 smithi053 ceph-mon[25205]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:43.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:35:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:42] "GET /metrics HTTP/1.1" 200 207938 "" "Prometheus/2.33.4" 2024-05-15T15:35:43.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:43 smithi043 ceph-mon[24430]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:43 smithi043 ceph-mon[29127]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:43.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:35:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:35:43.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:35:43.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:35:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:35:43.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:35:43.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:35:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:35:43.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:35:43.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:43 smithi053 ceph-mon[25205]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:35:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:35:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:35:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:35:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:35:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:35:44.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:35:44.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:35:44.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:35:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:35:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:35:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:35:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:45 smithi043 ceph-mon[24430]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:45 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:45 smithi043 ceph-mon[29127]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:45 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:45.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:45 smithi053 ceph-mon[25205]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:45 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:47.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:35:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:35:47.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:47 smithi043 ceph-mon[24430]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:47 smithi043 ceph-mon[29127]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:47.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:47 smithi053 ceph-mon[25205]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:49.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:49 smithi043 ceph-mon[24430]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:49 smithi043 ceph-mon[29127]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:49.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:49 smithi053 ceph-mon[25205]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:51.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:51 smithi043 ceph-mon[24430]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:51.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:51 smithi043 ceph-mon[29127]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:51.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:51 smithi053 ceph-mon[25205]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:53.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:35:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:52] "GET /metrics HTTP/1.1" 200 207947 "" "Prometheus/2.33.4" 2024-05-15T15:35:53.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:53 smithi043 ceph-mon[24430]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:53 smithi043 ceph-mon[29127]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:53.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:35:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:35:53.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:35:53.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:35:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:35:53.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:35:53.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:35:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:35:53.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:35:53.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:53 smithi053 ceph-mon[25205]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:55 smithi043 ceph-mon[24430]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:55 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:55 smithi043 ceph-mon[29127]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:55 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:55.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:55 smithi053 ceph-mon[25205]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:55 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:35:57.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:35:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[26314]: ::ffff:172.21.15.53 - - [15/May/2024:15:35:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:35:57.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:57 smithi043 ceph-mon[29127]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:57.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:57 smithi043 ceph-mon[24430]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:57.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:57 smithi053 ceph-mon[25205]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:35:59.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:35:59 smithi043 ceph-mon[24430]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:35:59 smithi043 ceph-mon[29127]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:35:59.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:35:59 smithi053 ceph-mon[25205]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:01.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:01 smithi043 ceph-mon[24430]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:01.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:01 smithi043 ceph-mon[29127]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:01.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:01 smithi053 ceph-mon[25205]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:03.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:36:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:02] "GET /metrics HTTP/1.1" 200 207958 "" "Prometheus/2.33.4" 2024-05-15T15:36:03.674 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:03 smithi053 ceph-mon[25205]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:03.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:03 smithi043 ceph-mon[24430]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:03 smithi043 ceph-mon[29127]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:03.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:36:03.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:36:03.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:03.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:36:03.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:03.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:36:04.080 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:03 smithi053 systemd[1]: Stopping Ceph mgr.x for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:36:04.080 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:03 smithi053 bash[45041]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr.x" found: no such container 2024-05-15T15:36:04.080 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:03 smithi053 podman[45052]: 2024-05-15 15:36:03.832790473 +0000 UTC m=+0.124755194 container died da9fa0ee69bfc75269a640473e18708dc35b44359f0b847e8a7c91b9fd4d8c96 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, name=centos-stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.component=centos-stream-container, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, ceph=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, release=754, RELEASE=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-05-15T15:36:04.347 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 podman[45052]: 2024-05-15 15:36:04.211020153 +0000 UTC m=+0.502984868 container cleanup da9fa0ee69bfc75269a640473e18708dc35b44359f0b847e8a7c91b9fd4d8c96 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, RELEASE=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-15T15:36:04.347 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 bash[45052]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x 2024-05-15T15:36:04.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 podman[45066]: 2024-05-15 15:36:04.345954462 +0000 UTC m=+0.508763253 container remove da9fa0ee69bfc75269a640473e18708dc35b44359f0b847e8a7c91b9fd4d8c96 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, architecture=x86_64, GIT_CLEAN=True, name=centos-stream, release=754, distribution-scope=public, io.buildah.version=1.19.8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vendor=Red Hat, Inc.) 2024-05-15T15:36:04.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-05-15T15:36:04.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 bash[45080]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr.x" found: no such container 2024-05-15T15:36:04.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-05-15T15:36:04.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 systemd[1]: Stopped Ceph mgr.x for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:36:04.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.x.service: Consumed 18.296s CPU time. 2024-05-15T15:36:04.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 systemd[1]: Starting Ceph mgr.x for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:36:05.070 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 podman[45177]: 2024-05-15 15:36:04.711887142 +0000 UTC m=+0.018412663 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:36:05.071 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:04 smithi053 podman[45177]: 2024-05-15 15:36:04.813673585 +0000 UTC m=+0.120199161 container create cfeda1bd207cf7e69dcc383e3e96120ed7dca0fc5974503d3c8f79bca9d884ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-05-15T15:36:05.071 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:05 smithi053 podman[45177]: 2024-05-15 15:36:05.041239845 +0000 UTC m=+0.347765351 container init cfeda1bd207cf7e69dcc383e3e96120ed7dca0fc5974503d3c8f79bca9d884ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True) 2024-05-15T15:36:05.071 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:05 smithi053 podman[45177]: 2024-05-15 15:36:05.04409283 +0000 UTC m=+0.350618337 container start cfeda1bd207cf7e69dcc383e3e96120ed7dca0fc5974503d3c8f79bca9d884ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-05-15T15:36:05.085 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:04 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:04.828Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T15:36:05.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:05 smithi053 ceph-mgr[45196]: -- 172.21.15.53:0/413403281 <== mon.2 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559f96dee5a0 con 0x559f96dc2c00 2024-05-15T15:36:05.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:05 smithi053 bash[45177]: cfeda1bd207cf7e69dcc383e3e96120ed7dca0fc5974503d3c8f79bca9d884ad 2024-05-15T15:36:05.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:05 smithi053 systemd[1]: Started Ceph mgr.x for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:36:05.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:05 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:05.212+0000 7f99df7dd200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:36:05.543 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[24430]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:05.543 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:05.543 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:05.543 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:05.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:36:05.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:36:05.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:36:05.544 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[29127]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:05.544 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:05.544 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:05.545 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:05.545 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:36:05.545 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:36:05.545 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:05 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:36:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:05 smithi053 ceph-mon[25205]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:05 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:05 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:05 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:05 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:36:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:05 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:36:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:05 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:36:05.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:05 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:05.415+0000 7f99df7dd200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:36:05.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:05 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:05.537+0000 7f99df7dd200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:36:06.209 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:05 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:05.895+0000 7f99df7dd200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:36:06.209 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:05 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:05.984+0000 7f99df7dd200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:36:06.767 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:06 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:06.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:06 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:06.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:06 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:07 smithi053 ceph-mon[25205]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:07.631 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:07 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:07.345+0000 7f99df7dd200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:36:07.631 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:07 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:07.551+0000 7f99df7dd200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:36:07.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:07 smithi043 ceph-mon[24430]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:07.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:07 smithi043 ceph-mon[29127]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:08.053 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:07 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:07.730+0000 7f99df7dd200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:36:08.376 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:08.050+0000 7f99df7dd200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:36:08.376 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:08.149+0000 7f99df7dd200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:36:08.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:08.374+0000 7f99df7dd200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:36:08.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:08.587+0000 7f99df7dd200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:36:09.520 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:09.227+0000 7f99df7dd200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:36:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:09 smithi053 ceph-mon[25205]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:09.785 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:09.696+0000 7f99df7dd200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:36:09.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:09 smithi043 ceph-mon[24430]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:09 smithi043 ceph-mon[29127]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:10.039 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:09.801+0000 7f99df7dd200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:36:10.040 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:10 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:10.019+0000 7f99df7dd200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-15T15:36:10.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:10 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:10.586+0000 7f99df7dd200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:36:10.943 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:10 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:10.684+0000 7f99df7dd200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:36:11.311 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:11 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:11.311 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:11 smithi053 ceph-mon[25205]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:11.311 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:11 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:11.312 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:10 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:10.942+0000 7f99df7dd200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:36:11.312 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:11.085+0000 7f99df7dd200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:36:11.312 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:11.185+0000 7f99df7dd200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:36:11.312 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:11.309+0000 7f99df7dd200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:36:11.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:11 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:11.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:11 smithi043 ceph-mon[24430]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:11.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:11 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:11.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:11 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:11.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:11 smithi043 ceph-mon[29127]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:11.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:11 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:36:11.879 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:11.623+0000 7f99df7dd200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:36:12.599 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:12 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:12.259+0000 7f99df7dd200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:36:12.600 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:12 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:12.489+0000 7f99df7dd200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:36:12.600 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:12 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:36:12] ENGINE Bus STARTING 2024-05-15T15:36:12.600 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:12 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: CherryPy Checker: 2024-05-15T15:36:12.600 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:12 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: The Application mounted at '' has an empty config. 2024-05-15T15:36:12.600 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:12 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:36:12.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:12 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:36:12] ENGINE Serving on http://:::9283 2024-05-15T15:36:12.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:12 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:36:12] ENGINE Bus STARTED 2024-05-15T15:36:13.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:36:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:12] "GET /metrics HTTP/1.1" 200 207958 "" "Prometheus/2.33.4" 2024-05-15T15:36:13.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:36:13.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T15:36:13.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:36:13.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:13.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:36:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[24430]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[24430]: Standby manager daemon x restarted 2024-05-15T15:36:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[24430]: Standby manager daemon x started 2024-05-15T15:36:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:36:13.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:36:13.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:36:13.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:36:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[29127]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[29127]: Standby manager daemon x restarted 2024-05-15T15:36:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[29127]: Standby manager daemon x started 2024-05-15T15:36:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:36:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:36:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:36:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:13 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:36:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:13 smithi053 ceph-mon[25205]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:13 smithi053 ceph-mon[25205]: Standby manager daemon x restarted 2024-05-15T15:36:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:13 smithi053 ceph-mon[25205]: Standby manager daemon x started 2024-05-15T15:36:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:13 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:36:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:13 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:36:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:13 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:36:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:13 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:36:14.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:14 smithi043 ceph-mon[24430]: mgrmap e21: y(active, since 6m), standbys: x 2024-05-15T15:36:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:14 smithi043 ceph-mon[29127]: mgrmap e21: y(active, since 6m), standbys: x 2024-05-15T15:36:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:14 smithi053 ceph-mon[25205]: mgrmap e21: y(active, since 6m), standbys: x 2024-05-15T15:36:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:15 smithi043 ceph-mon[24430]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:15 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:15 smithi043 ceph-mon[29127]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:15 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:15.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:15 smithi053 ceph-mon[25205]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:15 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:17.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:16 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:36:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:17 smithi053 ceph-mon[25205]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:17.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:17 smithi043 ceph-mon[24430]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:17 smithi043 ceph-mon[29127]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:19 smithi043 ceph-mon[24430]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:19 smithi043 ceph-mon[29127]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:19 smithi053 ceph-mon[25205]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:21.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:21 smithi043 ceph-mon[24430]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:21 smithi043 ceph-mon[29127]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:21 smithi053 ceph-mon[25205]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:23.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:36:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:22] "GET /metrics HTTP/1.1" 200 207957 "" "Prometheus/2.33.4" 2024-05-15T15:36:23.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:23 smithi043 ceph-mon[24430]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:23 smithi043 ceph-mon[29127]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:23.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:36:23.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:36:23.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:23.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:36:23.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:36:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:23 smithi053 ceph-mon[25205]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:25.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:25 smithi043 ceph-mon[24430]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:25 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:25 smithi043 ceph-mon[29127]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:25 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:25 smithi053 ceph-mon[25205]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:25 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:27.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:36:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:27 smithi043 ceph-mon[24430]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:27.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:27 smithi043 ceph-mon[29127]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:27.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:27 smithi053 ceph-mon[25205]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:29.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:29 smithi043 ceph-mon[24430]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:29 smithi043 ceph-mon[29127]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:29.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:29 smithi053 ceph-mon[25205]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:31.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:31 smithi043 ceph-mon[24430]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:31.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:31 smithi043 ceph-mon[29127]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:31.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:31 smithi053 ceph-mon[25205]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:33.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:36:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:32] "GET /metrics HTTP/1.1" 200 207952 "" "Prometheus/2.33.4" 2024-05-15T15:36:33.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:33 smithi043 ceph-mon[24430]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:33 smithi043 ceph-mon[29127]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:33.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:36:33.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:36:33.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:36:33.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:36:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:33 smithi053 ceph-mon[25205]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:35 smithi043 ceph-mon[24430]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:35 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:35 smithi043 ceph-mon[29127]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:35 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:35 smithi053 ceph-mon[25205]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:35 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:37.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:36:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:37 smithi043 ceph-mon[29127]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:37.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:37 smithi043 ceph-mon[24430]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:37.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:37 smithi053 ceph-mon[25205]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:39.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:39 smithi053 ceph-mon[25205]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:39 smithi043 ceph-mon[24430]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:39 smithi043 ceph-mon[29127]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:41.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:41 smithi043 ceph-mon[24430]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:41.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:41 smithi043 ceph-mon[29127]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:41.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:41 smithi053 ceph-mon[25205]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:43.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:36:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:42] "GET /metrics HTTP/1.1" 200 207952 "" "Prometheus/2.33.4" 2024-05-15T15:36:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:43 smithi043 ceph-mon[24430]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:43 smithi043 ceph-mon[29127]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:43.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:36:43.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:36:43.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:36:43.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:36:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:43 smithi053 ceph-mon[25205]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:44.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:36:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:36:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:36:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:36:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:36:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:36:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:36:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:36:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:36:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:36:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:36:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:36:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:45 smithi043 ceph-mon[24430]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:45 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:45 smithi043 ceph-mon[29127]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:45 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:45 smithi053 ceph-mon[25205]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:45 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:47.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:36:47.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:47 smithi043 ceph-mon[24430]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:47 smithi043 ceph-mon[29127]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:47 smithi053 ceph-mon[25205]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:49.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:49 smithi043 ceph-mon[24430]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:49 smithi043 ceph-mon[29127]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:49 smithi053 ceph-mon[25205]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:51.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:51 smithi043 ceph-mon[24430]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:51.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:51 smithi043 ceph-mon[29127]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:51 smithi053 ceph-mon[25205]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:53.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:36:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:52] "GET /metrics HTTP/1.1" 200 207950 "" "Prometheus/2.33.4" 2024-05-15T15:36:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:53 smithi043 ceph-mon[24430]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:53 smithi043 ceph-mon[29127]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:53.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:36:53.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:36:53.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:36:53.837 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:36:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:36:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:36:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:53 smithi053 ceph-mon[25205]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:55 smithi043 ceph-mon[24430]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:55 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:55 smithi043 ceph-mon[29127]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:55 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:55 smithi053 ceph-mon[25205]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:55 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:36:57.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:36:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:36:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:36:57.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:57 smithi043 ceph-mon[24430]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:57 smithi043 ceph-mon[29127]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:57 smithi053 ceph-mon[25205]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:36:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:36:59 smithi043 ceph-mon[24430]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:36:59 smithi043 ceph-mon[29127]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:36:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:36:59 smithi053 ceph-mon[25205]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:01 smithi043 ceph-mon[24430]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:01.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:01 smithi043 ceph-mon[29127]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:01 smithi053 ceph-mon[25205]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:03.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:37:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:02] "GET /metrics HTTP/1.1" 200 207953 "" "Prometheus/2.33.4" 2024-05-15T15:37:03.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:03 smithi043 ceph-mon[24430]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:03 smithi043 ceph-mon[29127]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:03.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:37:03.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:03.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:37:03.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:03 smithi053 ceph-mon[25205]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:05.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:05 smithi043 ceph-mon[24430]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:05 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:05 smithi043 ceph-mon[29127]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:05 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:05 smithi053 ceph-mon[25205]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:05 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:07.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:37:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:37:07.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:07 smithi043 ceph-mon[24430]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:07.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:07 smithi043 ceph-mon[29127]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:07 smithi053 ceph-mon[25205]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:09 smithi043 ceph-mon[24430]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:09 smithi043 ceph-mon[29127]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:09 smithi053 ceph-mon[25205]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:11 smithi043 ceph-mon[29127]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:11 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:37:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:11 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:37:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:11 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:37:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:11 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:37:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:11 smithi043 ceph-mon[24430]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:11 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:37:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:11 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:37:11.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:11 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:37:11.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:11 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:37:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:11 smithi053 ceph-mon[25205]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:11 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:37:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:11 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:37:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:11 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:37:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:11 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:37:13.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:37:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:12] "GET /metrics HTTP/1.1" 200 207953 "" "Prometheus/2.33.4" 2024-05-15T15:37:13.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:13 smithi043 ceph-mon[24430]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:13 smithi043 ceph-mon[29127]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:13.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:37:13.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:13.837 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:37:13.837 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:13 smithi053 ceph-mon[25205]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:15.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:15 smithi043 ceph-mon[24430]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:15 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:15 smithi043 ceph-mon[29127]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:15 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:15 smithi053 ceph-mon[25205]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:15 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:17.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:37:16 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:37:17.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:17 smithi043 ceph-mon[24430]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:17 smithi043 ceph-mon[29127]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:17 smithi053 ceph-mon[25205]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:19.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:19 smithi043 ceph-mon[24430]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:19 smithi043 ceph-mon[29127]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:19 smithi053 ceph-mon[25205]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:21.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:21 smithi043 ceph-mon[24430]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:21 smithi043 ceph-mon[29127]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:21 smithi053 ceph-mon[25205]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:23.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:37:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:22] "GET /metrics HTTP/1.1" 200 207961 "" "Prometheus/2.33.4" 2024-05-15T15:37:23.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:23 smithi043 ceph-mon[24430]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:23 smithi043 ceph-mon[29127]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:23.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:37:23.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:23.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:23.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:37:23.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:23.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:23 smithi053 ceph-mon[25205]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:25.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:25 smithi043 ceph-mon[29127]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:25.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:25 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:25 smithi043 ceph-mon[24430]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:25 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:25 smithi053 ceph-mon[25205]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:25 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:27.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:37:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:37:27.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:27 smithi043 ceph-mon[24430]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:27.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:27 smithi043 ceph-mon[29127]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:27 smithi053 ceph-mon[25205]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:29 smithi043 ceph-mon[24430]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:29 smithi043 ceph-mon[29127]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:29 smithi053 ceph-mon[25205]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:31.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:31 smithi043 ceph-mon[24430]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:31.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:31 smithi043 ceph-mon[29127]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:31.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:31 smithi053 ceph-mon[25205]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:33.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:37:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:32] "GET /metrics HTTP/1.1" 200 207963 "" "Prometheus/2.33.4" 2024-05-15T15:37:33.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:33 smithi043 ceph-mon[24430]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:33 smithi043 ceph-mon[29127]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:33.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:37:33.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:33.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:37:33.837 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:33.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:33 smithi053 ceph-mon[25205]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:35.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:35 smithi043 ceph-mon[24430]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:35 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:35 smithi043 ceph-mon[29127]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:35 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:35 smithi053 ceph-mon[25205]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:35 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:37.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:37:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:37:37.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:37 smithi043 ceph-mon[24430]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:37 smithi043 ceph-mon[29127]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:37 smithi053 ceph-mon[25205]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:39.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:39 smithi043 ceph-mon[24430]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:39 smithi043 ceph-mon[29127]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:39 smithi053 ceph-mon[25205]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:41.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:41 smithi043 ceph-mon[24430]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:41.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:41 smithi043 ceph-mon[29127]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:41 smithi053 ceph-mon[25205]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:43.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:37:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:42] "GET /metrics HTTP/1.1" 200 207963 "" "Prometheus/2.33.4" 2024-05-15T15:37:43.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:43 smithi043 ceph-mon[24430]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:43 smithi043 ceph-mon[29127]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:43.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:37:43.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:43.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:37:43.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:43 smithi053 ceph-mon[25205]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:37:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:44 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:37:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:37:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:44 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:37:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:37:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:44 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:37:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:37:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:44 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:37:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:37:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:44 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:37:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:37:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:44 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:37:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:45 smithi043 ceph-mon[29127]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:45 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:45 smithi043 ceph-mon[24430]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:45 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:45 smithi053 ceph-mon[25205]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:45 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:47.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:37:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:37:47.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:47 smithi043 ceph-mon[29127]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:47.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:47 smithi043 ceph-mon[24430]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:47 smithi053 ceph-mon[25205]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:49 smithi043 ceph-mon[29127]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:49 smithi043 ceph-mon[24430]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:49 smithi053 ceph-mon[25205]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:51.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:51 smithi043 ceph-mon[24430]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:51.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:51 smithi043 ceph-mon[29127]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:51 smithi053 ceph-mon[25205]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:53.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:37:52 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:52] "GET /metrics HTTP/1.1" 200 207961 "" "Prometheus/2.33.4" 2024-05-15T15:37:53.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:53 smithi043 ceph-mon[24430]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:53 smithi043 ceph-mon[29127]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:53.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:37:53.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:53.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:37:53.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:37:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:37:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:37:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:53 smithi053 ceph-mon[25205]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:55 smithi043 ceph-mon[29127]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:55 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:55 smithi043 ceph-mon[24430]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:55 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:55 smithi053 ceph-mon[25205]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:55 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:37:57.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:37:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:37:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:37:57.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:57 smithi043 ceph-mon[24430]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:57 smithi043 ceph-mon[29127]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:57 smithi053 ceph-mon[25205]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:37:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:37:59 smithi043 ceph-mon[24430]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:37:59 smithi043 ceph-mon[29127]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:37:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:37:59 smithi053 ceph-mon[25205]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:01.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:01 smithi043 ceph-mon[24430]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:01.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:01 smithi043 ceph-mon[29127]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:01.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:01 smithi053 ceph-mon[25205]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:01.916 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T15:38:02.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:02 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:38:02] "GET /metrics HTTP/1.1" 200 207956 "" "Prometheus/2.33.4" 2024-05-15T15:38:03.771 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:03 smithi043 ceph-mon[24430]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:03.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:03 smithi043 ceph-mon[29127]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:03.772 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:38:03.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:38:03.772 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:38:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:38:03.772 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:38:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:38:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:03 smithi053 ceph-mon[25205]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (6m) 2m ago 7m 24.1M - ba2b418f427c c7485334fa8c 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (6m) 113s ago 6m 56.3M - 8.3.5 dad864ee21e9 3e2dc1d42275 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 running (6m) 2m ago 6m 53.2M - 3.5 e1d6a67b021e 2a6c547e80fa 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283 running (119s) 113s ago 11m 67.9M - 19.0.0-2576-gde5210d5 48bc18eb22d8 cfeda1bd207c 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:9283 running (13m) 2m ago 13m 459M - 17.2.0 e1d6a67b021e faaa138d25ee 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (13m) 2m ago 13m 50.1M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (12m) 113s ago 12m 40.4M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (12m) 2m ago 12m 37.8M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (7m) 2m ago 7m 17.1M - 1dbe0e931976 8cfbd4656c29 2024-05-15T15:38:04.005 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (7m) 113s ago 7m 15.6M - 1dbe0e931976 c6fac2ae2513 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (11m) 2m ago 11m 49.3M 4017M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (11m) 2m ago 11m 52.7M 4017M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (10m) 2m ago 10m 46.7M 4017M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (10m) 2m ago 10m 48.5M 4017M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (9m) 113s ago 9m 47.9M 4273M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (9m) 113s ago 9m 46.2M 4273M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (9m) 113s ago 9m 48.7M 4273M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (8m) 113s ago 8m 51.1M 4273M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (6m) 113s ago 8m 64.3M - 514e6a882f6e 8e95fb39ec80 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (6m) 2m ago 6m 70.1M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:38:04.006 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (6m) 113s ago 6m 68.9M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:38:04.625 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions' 2024-05-15T15:38:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:04 smithi043 ceph-mon[24430]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:38:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:04 smithi043 ceph-mon[29127]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:38:04.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:04 smithi053 ceph-mon[25205]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:38:05.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:05 smithi043 ceph-mon[24430]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:05.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:05 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:05.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:05 smithi043 ceph-mon[29127]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:05.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:05 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:05 smithi053 ceph-mon[25205]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:05 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:06.831 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:38:06.832 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:38:06.832 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:38:06.832 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:38:06.832 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:38:06.832 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-15T15:38:06.832 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 1 2024-05-15T15:38:06.832 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:38:06.832 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:38:06.832 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:38:06.832 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:38:06.833 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:38:06.833 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:38:06.833 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:38:06.833 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:38:06.833 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:38:06.833 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-05-15T15:38:06.833 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 1 2024-05-15T15:38:06.833 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:38:06.833 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:38:07.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:38:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:38:07.425 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph -s' 2024-05-15T15:38:07.744 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:07 smithi043 ceph-mon[24430]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:07.744 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:07 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3161651510' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:38:07.744 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:07 smithi043 ceph-mon[29127]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:07.744 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:07 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3161651510' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:38:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:07 smithi053 ceph-mon[25205]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:07 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3161651510' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:38:09.407 INFO:teuthology.orchestra.run.smithi043.stdout: cluster: 2024-05-15T15:38:09.407 INFO:teuthology.orchestra.run.smithi043.stdout: id: 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:38:09.407 INFO:teuthology.orchestra.run.smithi043.stdout: health: HEALTH_OK 2024-05-15T15:38:09.407 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:38:09.407 INFO:teuthology.orchestra.run.smithi043.stdout: services: 2024-05-15T15:38:09.407 INFO:teuthology.orchestra.run.smithi043.stdout: mon: 3 daemons, quorum a,c,b (age 11m) 2024-05-15T15:38:09.407 INFO:teuthology.orchestra.run.smithi043.stdout: mgr: y(active, since 8m), standbys: x 2024-05-15T15:38:09.408 INFO:teuthology.orchestra.run.smithi043.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 8m) 2024-05-15T15:38:09.408 INFO:teuthology.orchestra.run.smithi043.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-15T15:38:09.408 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:38:09.408 INFO:teuthology.orchestra.run.smithi043.stdout: data: 2024-05-15T15:38:09.408 INFO:teuthology.orchestra.run.smithi043.stdout: pools: 6 pools, 161 pgs 2024-05-15T15:38:09.408 INFO:teuthology.orchestra.run.smithi043.stdout: objects: 209 objects, 585 KiB 2024-05-15T15:38:09.408 INFO:teuthology.orchestra.run.smithi043.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:09.408 INFO:teuthology.orchestra.run.smithi043.stdout: pgs: 161 active+clean 2024-05-15T15:38:09.408 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:38:09.408 INFO:teuthology.orchestra.run.smithi043.stdout: io: 2024-05-15T15:38:09.409 INFO:teuthology.orchestra.run.smithi043.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-15T15:38:09.409 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:38:09.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:09 smithi043 ceph-mon[24430]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:09.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:09 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/363048500' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:38:09.704 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:09 smithi043 ceph-mon[29127]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:09.704 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:09 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/363048500' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:38:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:09 smithi053 ceph-mon[25205]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:09 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/363048500' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:38:10.013 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T15:38:11.303 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:11 smithi043 ceph-mon[24430]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:11.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:11 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:38:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:11 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:11 smithi043 ceph-mon[24430]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:38:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:11 smithi043 ceph-mon[24430]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:38:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:11 smithi043 ceph-mon[29127]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:11 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:38:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:11 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:11 smithi043 ceph-mon[29127]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:38:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:11 smithi043 ceph-mon[29127]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:38:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:11 smithi053 ceph-mon[25205]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:11 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:38:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:11 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:11 smithi053 ceph-mon[25205]: from='mgr.24325 172.21.15.43:0/2432037412' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:38:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:11 smithi053 ceph-mon[25205]: from='mgr.24325 ' entity='mgr.y' 2024-05-15T15:38:12.113 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_OK 2024-05-15T15:38:12.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:12 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2016245641' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:38:12.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:12 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2016245641' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:38:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:12 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2016245641' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:38:12.886 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:38:12] "GET /metrics HTTP/1.1" 200 207956 "" "Prometheus/2.33.4" 2024-05-15T15:38:12.916 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-05-15T15:38:13.497 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:13 smithi043 ceph-mon[24430]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:13.497 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:13 smithi043 ceph-mon[29127]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:13 smithi053 ceph-mon[25205]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:13.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:38:13.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:38:13.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:38:13.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:38:13.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:38:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:38:14.873 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:38:15.507 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph mgr fail' 2024-05-15T15:38:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:15 smithi043 ceph-mon[24430]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:15 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/884779107' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:38:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:15 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:15 smithi043 ceph-mon[29127]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:15 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/884779107' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:38:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:15 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:15 smithi053 ceph-mon[25205]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:15 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/884779107' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:38:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:15 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:17.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:16 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:38:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:38:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:17 smithi043 ceph-mon[24430]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:17 smithi043 ceph-mon[29127]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:17 smithi053 ceph-mon[25205]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:18.711 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:18 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/666183118' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:38:18.711 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:18 smithi043 ceph-mon[29127]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:38:18.711 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:18 smithi043 ceph-mon[29127]: osdmap e96: 8 total, 8 up, 8 in 2024-05-15T15:38:18.711 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:18 smithi043 ceph-mon[29127]: Standby manager daemon y started 2024-05-15T15:38:18.712 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:18 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/666183118' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:38:18.712 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:18 smithi043 ceph-mon[24430]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:38:18.712 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:18 smithi043 ceph-mon[24430]: osdmap e96: 8 total, 8 up, 8 in 2024-05-15T15:38:18.712 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:18 smithi043 ceph-mon[24430]: Standby manager daemon y started 2024-05-15T15:38:18.712 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:18.474+0000 7f6226de5700 -1 mgr handle_mgr_map I was active but no longer amignoring --setuser ceph since I am not root 2024-05-15T15:38:18.712 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ignoring --setgroup ceph since I am not root 2024-05-15T15:38:18.712 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:18 smithi043 ceph-mgr[24660]: -- 172.21.15.43:0/3955330466 <== mon.1 v2:172.21.15.43:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5623b8698340 con 0x5623b8670800 2024-05-15T15:38:18.713 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:18.659+0000 7f415325c000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:38:18.713 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:18 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/666183118' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:38:18.713 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:18 smithi053 ceph-mon[25205]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:38:18.713 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:18 smithi053 ceph-mon[25205]: osdmap e96: 8 total, 8 up, 8 in 2024-05-15T15:38:18.713 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:18 smithi053 ceph-mon[25205]: Standby manager daemon y started 2024-05-15T15:38:18.713 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:18] ENGINE Bus STOPPING 2024-05-15T15:38:18.714 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-15T15:38:18.714 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:18] ENGINE Bus STOPPED 2024-05-15T15:38:18.714 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:18] ENGINE Bus STARTING 2024-05-15T15:38:18.714 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:18] ENGINE Serving on http://:::9283 2024-05-15T15:38:18.714 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:18] ENGINE Bus STARTED 2024-05-15T15:38:19.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:18.799+0000 7f415325c000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:38:19.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:18.905+0000 7f415325c000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:38:19.201 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'sleep 180' 2024-05-15T15:38:19.468 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:19.251+0000 7f415325c000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:38:19.468 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:19.340+0000 7f415325c000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:38:19.728 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: mgrmap e22: x(active, starting, since 0.999683s), standbys: y 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:38:19.729 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:38:19.730 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:38:19.730 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:38:19.730 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:38:19.730 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:38:19.730 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: Manager daemon x is now available 2024-05-15T15:38:19.730 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:19.730 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: Queued rgw.foo for migration 2024-05-15T15:38:19.730 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-15T15:38:19.733 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:19.733 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:38:19.733 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:38:19.733 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:38:19.733 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:38:19.734 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:38:19.734 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:38:19.734 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: Deploying cephadm binary to smithi043 2024-05-15T15:38:19.734 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[29127]: Deploying cephadm binary to smithi053 2024-05-15T15:38:19.734 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:38:19.734 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-15T15:38:19.734 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:38:19.734 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: mgrmap e22: x(active, starting, since 0.999683s), standbys: y 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:38:19.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: Manager daemon x is now available 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: Queued rgw.foo for migration 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: Deploying cephadm binary to smithi043 2024-05-15T15:38:19.736 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:19 smithi043 ceph-mon[24430]: Deploying cephadm binary to smithi053 2024-05-15T15:38:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:38:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: mgrmap e22: x(active, starting, since 0.999683s), standbys: y 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:38:19.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: Manager daemon x is now available 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: Queued rgw.foo for migration 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:38:19.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: Deploying cephadm binary to smithi043 2024-05-15T15:38:19.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:19 smithi053 ceph-mon[25205]: Deploying cephadm binary to smithi053 2024-05-15T15:38:20.335 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:20 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:38:20.079Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T15:38:20.754 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:20 smithi043 ceph-mon[24430]: mgrmap e23: x(active, since 2s), standbys: y 2024-05-15T15:38:20.754 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:20 smithi043 ceph-mon[24430]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:20.754 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:20 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:20.754 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:20 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:20.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:20 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:20.574+0000 7f415325c000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:38:20.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:20 smithi043 ceph-mon[29127]: mgrmap e23: x(active, since 2s), standbys: y 2024-05-15T15:38:20.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:20 smithi043 ceph-mon[29127]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:20.758 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:20 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:20.758 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:20 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:20.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:20 smithi053 ceph-mon[25205]: mgrmap e23: x(active, since 2s), standbys: y 2024-05-15T15:38:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:20 smithi053 ceph-mon[25205]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:20 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:20 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:21.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:20 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:20.754+0000 7f415325c000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:38:21.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:20 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:20.844+0000 7f415325c000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:38:21.486 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:21.174+0000 7f415325c000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:38:21.486 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:21.270+0000 7f415325c000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:38:21.486 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:21.370+0000 7f415325c000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:38:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:21 smithi043 ceph-mon[29127]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:21 smithi043 ceph-mon[29127]: [15/May/2024:15:38:20] ENGINE Bus STARTING 2024-05-15T15:38:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:21 smithi043 ceph-mon[29127]: [15/May/2024:15:38:21] ENGINE Serving on http://172.21.15.53:8765 2024-05-15T15:38:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:21 smithi043 ceph-mon[24430]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:21 smithi043 ceph-mon[24430]: [15/May/2024:15:38:20] ENGINE Bus STARTING 2024-05-15T15:38:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:21 smithi043 ceph-mon[24430]: [15/May/2024:15:38:21] ENGINE Serving on http://172.21.15.53:8765 2024-05-15T15:38:21.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:21.629+0000 7f415325c000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:38:21.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:21 smithi053 ceph-mon[25205]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:21 smithi053 ceph-mon[25205]: [15/May/2024:15:38:20] ENGINE Bus STARTING 2024-05-15T15:38:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:21 smithi053 ceph-mon[25205]: [15/May/2024:15:38:21] ENGINE Serving on http://172.21.15.53:8765 2024-05-15T15:38:22.494 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:22.102+0000 7f415325c000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:38:22.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:22 smithi043 ceph-mon[24430]: [15/May/2024:15:38:21] ENGINE Serving on https://172.21.15.53:7150 2024-05-15T15:38:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:22 smithi043 ceph-mon[24430]: [15/May/2024:15:38:21] ENGINE Bus STARTED 2024-05-15T15:38:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:22 smithi043 ceph-mon[24430]: mgrmap e24: x(active, since 4s), standbys: y 2024-05-15T15:38:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:22 smithi043 ceph-mon[29127]: [15/May/2024:15:38:21] ENGINE Serving on https://172.21.15.53:7150 2024-05-15T15:38:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:22 smithi043 ceph-mon[29127]: [15/May/2024:15:38:21] ENGINE Bus STARTED 2024-05-15T15:38:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:22 smithi043 ceph-mon[29127]: mgrmap e24: x(active, since 4s), standbys: y 2024-05-15T15:38:22.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:22.626+0000 7f415325c000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:38:22.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:22.726+0000 7f415325c000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:38:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:22 smithi053 ceph-mon[25205]: [15/May/2024:15:38:21] ENGINE Serving on https://172.21.15.53:7150 2024-05-15T15:38:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:22 smithi053 ceph-mon[25205]: [15/May/2024:15:38:21] ENGINE Bus STARTED 2024-05-15T15:38:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:22 smithi053 ceph-mon[25205]: mgrmap e24: x(active, since 4s), standbys: y 2024-05-15T15:38:23.335 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:23.039+0000 7f415325c000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:38:23.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:23.134+0000 7f415325c000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:38:23.808 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:23 smithi043 ceph-mon[24430]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:23.808 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:23 smithi043 ceph-mon[29127]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:23.809 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:23.564+0000 7f415325c000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:38:23.809 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:23.705+0000 7f415325c000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:38:23.809 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:38:23.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:38:23.809 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:38:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T15:38:23.809 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:38:23.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:38:23.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:23 smithi053 ceph-mon[25205]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:24.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:23.806+0000 7f415325c000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:38:24.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:23.931+0000 7f415325c000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:38:24.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:24.033+0000 7f415325c000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:38:24.807 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:24 smithi053 ceph-mon[25205]: mgrmap e25: x(active, since 6s), standbys: y 2024-05-15T15:38:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:24 smithi043 ceph-mon[24430]: mgrmap e25: x(active, since 6s), standbys: y 2024-05-15T15:38:24.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:24 smithi043 ceph-mon[29127]: mgrmap e25: x(active, since 6s), standbys: y 2024-05-15T15:38:24.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:24.639+0000 7f415325c000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:38:25.137 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:25.026+0000 7f415325c000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:38:25.137 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:38:25] ENGINE Bus STARTING 2024-05-15T15:38:25.137 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: CherryPy Checker: 2024-05-15T15:38:25.137 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: The Application mounted at '' has an empty config. 2024-05-15T15:38:25.137 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: 2024-05-15T15:38:25.137 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:38:25] ENGINE Serving on http://:::9283 2024-05-15T15:38:25.137 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: [15/May/2024:15:38:25] ENGINE Bus STARTED 2024-05-15T15:38:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:25 smithi053 ceph-mon[25205]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:25 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:25 smithi053 ceph-mon[25205]: Standby manager daemon y restarted 2024-05-15T15:38:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:25 smithi053 ceph-mon[25205]: Standby manager daemon y started 2024-05-15T15:38:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:25 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-15T15:38:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:25 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:38:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:25 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-15T15:38:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:25 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:38:25.800 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:38:25.617Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=4 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:38:25.800 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[29127]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:25.800 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:25.800 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[29127]: Standby manager daemon y restarted 2024-05-15T15:38:25.800 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[29127]: Standby manager daemon y started 2024-05-15T15:38:25.800 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-15T15:38:25.800 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:38:25.800 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-15T15:38:25.801 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:38:25.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[24430]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:38:25.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:25.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[24430]: Standby manager daemon y restarted 2024-05-15T15:38:25.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[24430]: Standby manager daemon y started 2024-05-15T15:38:25.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-15T15:38:25.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:38:25.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-15T15:38:25.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:25 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/1896109846' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:38:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:26 smithi043 ceph-mon[24430]: mgrmap e26: x(active, since 8s), standbys: y 2024-05-15T15:38:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:26 smithi043 ceph-mon[29127]: mgrmap e26: x(active, since 8s), standbys: y 2024-05-15T15:38:26.837 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:26 smithi053 ceph-mon[25205]: mgrmap e26: x(active, since 8s), standbys: y 2024-05-15T15:38:27.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:38:26] "GET /metrics HTTP/1.1" 200 35092 "" "Prometheus/2.33.4" 2024-05-15T15:38:27.829 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:27 smithi053 ceph-mon[25205]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:38:27.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:27 smithi043 ceph-mon[24430]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:38:27.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:27 smithi043 ceph-mon[29127]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:38:29.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:28 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:28 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:28 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:28 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:28 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:28 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:28 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:28 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:28 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:28 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:28 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:29.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:28 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: Adjusting osd_memory_target on smithi043 to 2993M 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: Adjusting osd_memory_target on smithi043 to 2993M 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:30.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:29 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:38:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-15T15:38:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: Adjusting osd_memory_target on smithi043 to 2993M 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-15T15:38:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:30.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:30.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:29 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:38:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: Adjusting osd_memory_target on smithi053 to 3249M 2024-05-15T15:38:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:38:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:38:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:38:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:38:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:38:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:38:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: Adjusting osd_memory_target on smithi053 to 3249M 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:38:31.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:30 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: Adjusting osd_memory_target on smithi053 to 3249M 2024-05-15T15:38:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:38:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:38:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:38:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:38:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:38:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:38:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:30 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:31.514 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 systemd[1]: Stopping Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:38:31.514 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 bash[57947]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter.a" found: no such container 2024-05-15T15:38:31.514 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 podman[57957]: 2024-05-15 15:38:31.255538434 +0000 UTC m=+0.121222022 container died 8cfbd4656c29d3dea493c8e3a39ca725039c920d6246582e3d5836872c449b35 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:31.807 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 podman[57957]: 2024-05-15 15:38:31.619939969 +0000 UTC m=+0.485623575 container cleanup 8cfbd4656c29d3dea493c8e3a39ca725039c920d6246582e3d5836872c449b35 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:31.807 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 bash[57957]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a 2024-05-15T15:38:31.807 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 bash[57984]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter.a" found: no such container 2024-05-15T15:38:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:31 smithi043 ceph-mon[24430]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:38:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:31 smithi043 ceph-mon[24430]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:38:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:31 smithi043 ceph-mon[24430]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:38:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:31 smithi043 ceph-mon[24430]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-15T15:38:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:31 smithi043 ceph-mon[24430]: Reconfiguring daemon node-exporter.a on smithi043 2024-05-15T15:38:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:31 smithi043 ceph-mon[29127]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:38:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:31 smithi043 ceph-mon[29127]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:38:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:31 smithi043 ceph-mon[29127]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:38:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:31 smithi043 ceph-mon[29127]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-15T15:38:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:31 smithi043 ceph-mon[29127]: Reconfiguring daemon node-exporter.a on smithi043 2024-05-15T15:38:32.088 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 podman[57972]: 2024-05-15 15:38:31.887938465 +0000 UTC m=+0.635450316 container remove 8cfbd4656c29d3dea493c8e3a39ca725039c920d6246582e3d5836872c449b35 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:32.088 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-15T15:38:32.088 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-15T15:38:32.088 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 systemd[1]: Stopped Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:38:32.088 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.a.service: Consumed 3.916s CPU time. 2024-05-15T15:38:32.088 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:31 smithi043 systemd[1]: Starting Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:38:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:31 smithi053 ceph-mon[25205]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:38:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:31 smithi053 ceph-mon[25205]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:38:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:31 smithi053 ceph-mon[25205]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:38:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:31 smithi053 ceph-mon[25205]: Reconfiguring node-exporter.a (dependencies changed)... 2024-05-15T15:38:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:31 smithi053 ceph-mon[25205]: Reconfiguring daemon node-exporter.a on smithi043 2024-05-15T15:38:32.426 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 podman[58048]: 2024-05-15 15:38:32.023697792 +0000 UTC m=+0.017333680 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-15T15:38:32.426 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 podman[58048]: 2024-05-15 15:38:32.147504524 +0000 UTC m=+0.141140453 container create 083a05fd38330bc76eb2de142de1d8f8638959775f07ede3994c9bf50a34f17d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:32.426 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 podman[58048]: 2024-05-15 15:38:32.424946105 +0000 UTC m=+0.418581984 container init 083a05fd38330bc76eb2de142de1d8f8638959775f07ede3994c9bf50a34f17d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:32.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:38:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:38:32.836 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 podman[58048]: 2024-05-15 15:38:32.428182517 +0000 UTC m=+0.421818405 container start 083a05fd38330bc76eb2de142de1d8f8638959775f07ede3994c9bf50a34f17d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:32.836 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.432Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-15T15:38:32.836 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.432Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-15T15:38:32.836 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.432Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-15T15:38:32.836 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.432Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=arp 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=edac 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-15T15:38:32.837 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-15T15:38:32.838 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=os 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=stat 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=time 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-15T15:38:32.839 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=uname 2024-05-15T15:38:32.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-15T15:38:32.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-15T15:38:32.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-15T15:38:32.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.433Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-15T15:38:32.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[58062]: ts=2024-05-15T15:38:32.434Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-15T15:38:32.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 bash[58048]: 083a05fd38330bc76eb2de142de1d8f8638959775f07ede3994c9bf50a34f17d 2024-05-15T15:38:32.840 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:38:32 smithi043 systemd[1]: Started Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:38:33.730 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:33 smithi043 ceph-mon[24430]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:38:33.731 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:33 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:33.731 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:33 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:33.731 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:33 smithi043 ceph-mon[24430]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-15T15:38:33.731 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:33 smithi043 ceph-mon[24430]: Reconfiguring daemon alertmanager.a on smithi043 2024-05-15T15:38:33.731 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:33 smithi043 ceph-mon[29127]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:38:33.731 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:33 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:33.731 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:33 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:33.731 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:33 smithi043 ceph-mon[29127]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-15T15:38:33.731 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:33 smithi043 ceph-mon[29127]: Reconfiguring daemon alertmanager.a on smithi043 2024-05-15T15:38:33.732 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=error ts=2024-05-15T15:38:33.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:38:33.732 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:38:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.43:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.43 because it doesn't contain any IP SANs" 2024-05-15T15:38:33.732 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=warn ts=2024-05-15T15:38:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.53:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.53 because it doesn't contain any IP SANs" 2024-05-15T15:38:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:33 smithi053 ceph-mon[25205]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:38:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:33 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:33 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:33 smithi053 ceph-mon[25205]: Reconfiguring alertmanager.a (dependencies changed)... 2024-05-15T15:38:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:33 smithi053 ceph-mon[25205]: Reconfiguring daemon alertmanager.a on smithi043 2024-05-15T15:38:34.652 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:34 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:38:34.652 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:34 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:38:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:34 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:38:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:35 smithi043 ceph-mon[29127]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:38:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:35 smithi043 ceph-mon[29127]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:35 smithi043 ceph-mon[24430]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:38:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:35 smithi043 ceph-mon[24430]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:35.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:35 smithi043 systemd[1]: Stopping Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:38:35.837 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:35 smithi043 bash[58233]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager.a" found: no such container 2024-05-15T15:38:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:35 smithi053 ceph-mon[25205]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:38:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:35 smithi053 ceph-mon[25205]: from='client.24647 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:36.181 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[49015]: level=info ts=2024-05-15T15:38:35.895Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-15T15:38:36.181 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:35 smithi043 podman[58245]: 2024-05-15 15:38:35.915941439 +0000 UTC m=+0.156506262 container died c7485334fa8cac3eef23551d6b5ef8b1c69e93d770ba1f6ab7be112abb90ce47 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:36.573 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 podman[58245]: 2024-05-15 15:38:36.270333432 +0000 UTC m=+0.510898255 container cleanup c7485334fa8cac3eef23551d6b5ef8b1c69e93d770ba1f6ab7be112abb90ce47 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:36.573 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 bash[58245]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a 2024-05-15T15:38:36.573 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 bash[58271]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager.a" found: no such container 2024-05-15T15:38:36.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 podman[58259]: 2024-05-15 15:38:36.571533686 +0000 UTC m=+0.650692936 container remove c7485334fa8cac3eef23551d6b5ef8b1c69e93d770ba1f6ab7be112abb90ce47 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:36.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 podman[58259]: 2024-05-15 15:38:36.647251553 +0000 UTC m=+0.726410800 volume remove bab6f291297a988433f59eda2493190701b25f779867e43cecc3aefa660afcfb 2024-05-15T15:38:36.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-15T15:38:36.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 systemd[1]: Stopped Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:38:36.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@alertmanager.a.service: Consumed 1.084s CPU time. 2024-05-15T15:38:36.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 systemd[1]: Starting Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:38:37.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:38:36] "GET /metrics HTTP/1.1" 200 37885 "" "Prometheus/2.33.4" 2024-05-15T15:38:37.208 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 podman[58332]: 2024-05-15 15:38:36.79984217 +0000 UTC m=+0.021191941 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-05-15T15:38:37.208 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:36 smithi043 podman[58332]: 2024-05-15 15:38:36.931804662 +0000 UTC m=+0.153154433 volume create 8d4ccbef251e4cfa12982f63fcbbd8b14d0948f15b50135c3267d232be9d5007 2024-05-15T15:38:37.208 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 podman[58332]: 2024-05-15 15:38:37.032199366 +0000 UTC m=+0.253549134 container create dd5a5b0f3fe5f39f7907eec560ffd624568ac52824e2279aa483414fa4913c4d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:37.550 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 podman[58332]: 2024-05-15 15:38:37.318424742 +0000 UTC m=+0.539774511 container init dd5a5b0f3fe5f39f7907eec560ffd624568ac52824e2279aa483414fa4913c4d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:37.550 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 podman[58332]: 2024-05-15 15:38:37.321742185 +0000 UTC m=+0.543091963 container start dd5a5b0f3fe5f39f7907eec560ffd624568ac52824e2279aa483414fa4913c4d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:37.550 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:38:37.334Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-05-15T15:38:37.550 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:38:37.334Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-05-15T15:38:37.550 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:38:37.336Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.43 port=9094 2024-05-15T15:38:37.550 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:38:37.337Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-15T15:38:37.550 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:38:37.365Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-15T15:38:37.550 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:38:37.365Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-15T15:38:37.551 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:38:37.368Z caller=main.go:518 msg=Listening address=:9093 2024-05-15T15:38:37.551 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:38:37.368Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-05-15T15:38:37.551 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 bash[58332]: dd5a5b0f3fe5f39f7907eec560ffd624568ac52824e2279aa483414fa4913c4d 2024-05-15T15:38:37.551 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:37 smithi043 systemd[1]: Started Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:38:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[29127]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:38:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:38:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:38:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:37.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[24430]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:38:37.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:37.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:37.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:38:37.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:38:37.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:37 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:37 smithi053 ceph-mon[25205]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:38:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:37 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:37 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:37 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:38:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:37 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:38:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:37 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:38.752 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:38 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:38:38.752 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:38 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:38:38.752 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:38 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:38:38.752 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:38 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:38:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:38 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:38:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:38 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:38:39.629 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:39 smithi043 ceph-mon[24430]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:39.630 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:39 smithi043 ceph-mon[29127]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:39.630 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:38:39.338Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000788324s 2024-05-15T15:38:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:39 smithi053 ceph-mon[25205]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:41.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:41 smithi043 ceph-mon[24430]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:41.568 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:41 smithi043 ceph-mon[29127]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:41 smithi053 ceph-mon[25205]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:38:42.570 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.53:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.43:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:38:42.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:42 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:42 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:42 smithi053 ceph-mon[25205]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:38:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:42 smithi053 ceph-mon[25205]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:38:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:42 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/4243343927' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T15:38:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:42 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1771812597' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3889110225"}]: dispatch 2024-05-15T15:38:43.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:43.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:43.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[24430]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:38:43.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[24430]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:38:43.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/4243343927' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T15:38:43.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1771812597' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3889110225"}]: dispatch 2024-05-15T15:38:43.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:38:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[24656]: ::ffff:172.21.15.53 - - [15/May/2024:15:38:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:38:43.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:43.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:43.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[29127]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:38:43.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[29127]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:38:43.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/4243343927' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T15:38:43.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:42 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1771812597' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3889110225"}]: dispatch 2024-05-15T15:38:43.862 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:43 smithi053 ceph-mon[25205]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:43.862 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:43 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1771812597' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3889110225"}]': finished 2024-05-15T15:38:43.862 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:43 smithi053 ceph-mon[25205]: osdmap e97: 8 total, 8 up, 8 in 2024-05-15T15:38:43.862 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:43 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2253086060' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3889110225"}]: dispatch 2024-05-15T15:38:43.862 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:43 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3889110225"}]: dispatch 2024-05-15T15:38:44.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:43 smithi043 ceph-mon[24430]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:44.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:43 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1771812597' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3889110225"}]': finished 2024-05-15T15:38:44.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:43 smithi043 ceph-mon[24430]: osdmap e97: 8 total, 8 up, 8 in 2024-05-15T15:38:44.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:43 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2253086060' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3889110225"}]: dispatch 2024-05-15T15:38:44.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:43 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3889110225"}]: dispatch 2024-05-15T15:38:44.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:43 smithi043 ceph-mon[29127]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:44.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:43 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1771812597' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/3889110225"}]': finished 2024-05-15T15:38:44.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:43 smithi043 ceph-mon[29127]: osdmap e97: 8 total, 8 up, 8 in 2024-05-15T15:38:44.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:43 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2253086060' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3889110225"}]: dispatch 2024-05-15T15:38:44.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:43 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3889110225"}]: dispatch 2024-05-15T15:38:44.969 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:44 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3889110225"}]': finished 2024-05-15T15:38:44.970 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:44 smithi053 ceph-mon[25205]: osdmap e98: 8 total, 8 up, 8 in 2024-05-15T15:38:44.970 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:44 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1231488575' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/699734202"}]: dispatch 2024-05-15T15:38:45.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:44 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3889110225"}]': finished 2024-05-15T15:38:45.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:44 smithi043 ceph-mon[24430]: osdmap e98: 8 total, 8 up, 8 in 2024-05-15T15:38:45.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:44 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1231488575' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/699734202"}]: dispatch 2024-05-15T15:38:45.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:44 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/3889110225"}]': finished 2024-05-15T15:38:45.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:44 smithi043 ceph-mon[29127]: osdmap e98: 8 total, 8 up, 8 in 2024-05-15T15:38:45.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:44 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1231488575' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/699734202"}]: dispatch 2024-05-15T15:38:45.380 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 systemd[1]: Stopping Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:38:45.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 bash[48217]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus.a" found: no such container 2024-05-15T15:38:45.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.181Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-15T15:38:45.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.181Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-15T15:38:45.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.181Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-15T15:38:45.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.181Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-15T15:38:45.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.181Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-15T15:38:45.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.181Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-15T15:38:45.381 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.181Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-15T15:38:45.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.181Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-15T15:38:45.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.181Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-15T15:38:45.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.182Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-15T15:38:45.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.182Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-15T15:38:45.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[41942]: ts=2024-05-15T15:38:45.182Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-15T15:38:45.382 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 podman[48228]: 2024-05-15 15:38:45.191668776 +0000 UTC m=+0.132626889 container died 8e95fb39ec80e823bbfcda958858016688b5935a6df44f442d831e1041731b6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:45.791 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 podman[48228]: 2024-05-15 15:38:45.59722029 +0000 UTC m=+0.538178392 container cleanup 8e95fb39ec80e823bbfcda958858016688b5935a6df44f442d831e1041731b6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:45.791 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 bash[48228]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a 2024-05-15T15:38:45.791 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:45 smithi053 ceph-mon[25205]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-15T15:38:45.792 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:45 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1231488575' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/699734202"}]': finished 2024-05-15T15:38:45.792 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:45 smithi053 ceph-mon[25205]: osdmap e99: 8 total, 8 up, 8 in 2024-05-15T15:38:45.792 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:45 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3913203014' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4085531265"}]: dispatch 2024-05-15T15:38:45.792 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:45 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4085531265"}]: dispatch 2024-05-15T15:38:45.987 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:45 smithi043 ceph-mon[29127]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-15T15:38:45.987 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:45 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1231488575' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/699734202"}]': finished 2024-05-15T15:38:45.987 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:45 smithi043 ceph-mon[29127]: osdmap e99: 8 total, 8 up, 8 in 2024-05-15T15:38:45.987 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:45 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3913203014' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4085531265"}]: dispatch 2024-05-15T15:38:45.987 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:45 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4085531265"}]: dispatch 2024-05-15T15:38:45.988 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:45 smithi043 ceph-mon[24430]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-05-15T15:38:45.988 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:45 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1231488575' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/699734202"}]': finished 2024-05-15T15:38:45.988 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:45 smithi043 ceph-mon[24430]: osdmap e99: 8 total, 8 up, 8 in 2024-05-15T15:38:45.988 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:45 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3913203014' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4085531265"}]: dispatch 2024-05-15T15:38:45.988 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:45 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4085531265"}]: dispatch 2024-05-15T15:38:46.065 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 podman[48243]: 2024-05-15 15:38:45.790324171 +0000 UTC m=+0.595702035 container remove 8e95fb39ec80e823bbfcda958858016688b5935a6df44f442d831e1041731b6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:46.065 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 bash[48258]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus.a" found: no such container 2024-05-15T15:38:46.065 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-15T15:38:46.065 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 systemd[1]: Stopped Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:38:46.065 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Consumed 2.026s CPU time. 2024-05-15T15:38:46.065 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:45 smithi053 systemd[1]: Starting Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 podman[48320]: 2024-05-15 15:38:45.968028689 +0000 UTC m=+0.019218143 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 podman[48320]: 2024-05-15 15:38:46.102791585 +0000 UTC m=+0.153981083 container create e8019f2a645644695438de85f667ba88a59ad2d43c2569a276f7fe95992ec296 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 podman[48320]: 2024-05-15 15:38:46.285397766 +0000 UTC m=+0.336587220 container init e8019f2a645644695438de85f667ba88a59ad2d43c2569a276f7fe95992ec296 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 podman[48320]: 2024-05-15 15:38:46.288420275 +0000 UTC m=+0.339609719 container start e8019f2a645644695438de85f667ba88a59ad2d43c2569a276f7fe95992ec296 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.312Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.312Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.313Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.313Z caller=main.go:518 level=info host_details="(Linux 5.14.0-446.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024 x86_64 smithi053 (none))" 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.313Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.313Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-15T15:38:46.317 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.315Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-15T15:38:46.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.316Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-15T15:38:46.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.317Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-15T15:38:46.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.319Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-15T15:38:46.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.319Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.669µs 2024-05-15T15:38:46.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.319Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-15T15:38:46.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 bash[48320]: e8019f2a645644695438de85f667ba88a59ad2d43c2569a276f7fe95992ec296 2024-05-15T15:38:46.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 systemd[1]: Started Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:38:46.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.543Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-05-15T15:38:46.985 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:46 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4085531265"}]': finished 2024-05-15T15:38:46.985 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:46 smithi053 ceph-mon[25205]: osdmap e100: 8 total, 8 up, 8 in 2024-05-15T15:38:46.985 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:46 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1741009251' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4292877123"}]: dispatch 2024-05-15T15:38:46.985 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:46 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4292877123"}]: dispatch 2024-05-15T15:38:46.985 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:46 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:46.985 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:46 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:46.986 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.788Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-05-15T15:38:46.986 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.788Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-05-15T15:38:46.986 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.788Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.591µs wal_replay_duration=469.228642ms total_replay_duration=469.274254ms 2024-05-15T15:38:46.986 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.791Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-15T15:38:46.986 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.791Z caller=main.go:947 level=info msg="TSDB started" 2024-05-15T15:38:46.986 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.791Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-15T15:38:46.986 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.815Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.798636ms db_storage=731ns remote_storage=1.515µs web_handler=315ns query_engine=655ns scrape=372.637µs scrape_sd=107.502µs notify=23.009µs notify_sd=23.284µs rules=22.780367ms 2024-05-15T15:38:46.986 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:46.815Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-15T15:38:46.997 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4085531265"}]': finished 2024-05-15T15:38:46.997 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[24430]: osdmap e100: 8 total, 8 up, 8 in 2024-05-15T15:38:46.997 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1741009251' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4292877123"}]: dispatch 2024-05-15T15:38:46.997 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4292877123"}]: dispatch 2024-05-15T15:38:46.997 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:46.997 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:46.998 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4085531265"}]': finished 2024-05-15T15:38:46.998 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[29127]: osdmap e100: 8 total, 8 up, 8 in 2024-05-15T15:38:46.998 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1741009251' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4292877123"}]: dispatch 2024-05-15T15:38:46.998 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4292877123"}]: dispatch 2024-05-15T15:38:46.998 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:46.998 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:46 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:47.380 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:46 smithi053 systemd[1]: Stopping Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:38:47.380 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:47 smithi053 bash[48445]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter.b" found: no such container 2024-05-15T15:38:47.714 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:38:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:38:47.340Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003383785s 2024-05-15T15:38:47.715 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:47 smithi053 podman[48458]: 2024-05-15 15:38:47.422349579 +0000 UTC m=+0.406328435 container died c6fac2ae2513753713036c847f9891b539cad8ab0c52331cf5d0e676ef46c469 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:38:48.024 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:47 smithi053 podman[48458]: 2024-05-15 15:38:47.838837179 +0000 UTC m=+0.822816025 container cleanup c6fac2ae2513753713036c847f9891b539cad8ab0c52331cf5d0e676ef46c469 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:38:48.024 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:47 smithi053 bash[48458]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b 2024-05-15T15:38:48.025 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:47 smithi053 bash[48483]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter.b" found: no such container 2024-05-15T15:38:48.025 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:47 smithi053 ceph-mon[25205]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-15T15:38:48.025 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:47 smithi053 ceph-mon[25205]: Reconfiguring daemon node-exporter.b on smithi053 2024-05-15T15:38:48.025 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:47 smithi053 ceph-mon[25205]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T15:38:48.025 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:47 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4292877123"}]': finished 2024-05-15T15:38:48.025 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:47 smithi053 ceph-mon[25205]: osdmap e101: 8 total, 8 up, 8 in 2024-05-15T15:38:48.025 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:47 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2013533699' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1300788763"}]: dispatch 2024-05-15T15:38:48.025 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:47 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1300788763"}]: dispatch 2024-05-15T15:38:48.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[24430]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-15T15:38:48.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[24430]: Reconfiguring daemon node-exporter.b on smithi053 2024-05-15T15:38:48.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[24430]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T15:38:48.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4292877123"}]': finished 2024-05-15T15:38:48.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[24430]: osdmap e101: 8 total, 8 up, 8 in 2024-05-15T15:38:48.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2013533699' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1300788763"}]: dispatch 2024-05-15T15:38:48.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1300788763"}]: dispatch 2024-05-15T15:38:48.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[29127]: Reconfiguring node-exporter.b (dependencies changed)... 2024-05-15T15:38:48.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[29127]: Reconfiguring daemon node-exporter.b on smithi053 2024-05-15T15:38:48.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[29127]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T15:38:48.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/4292877123"}]': finished 2024-05-15T15:38:48.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[29127]: osdmap e101: 8 total, 8 up, 8 in 2024-05-15T15:38:48.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2013533699' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1300788763"}]: dispatch 2024-05-15T15:38:48.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:47 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1300788763"}]: dispatch 2024-05-15T15:38:48.284 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 podman[48471]: 2024-05-15 15:38:48.023505241 +0000 UTC m=+0.603822690 container remove c6fac2ae2513753713036c847f9891b539cad8ab0c52331cf5d0e676ef46c469 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:38:48.284 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-15T15:38:48.284 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-15T15:38:48.284 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 systemd[1]: Stopped Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:38:48.284 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.b.service: Consumed 4.000s CPU time. 2024-05-15T15:38:48.284 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 systemd[1]: Starting Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:38:48.284 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 podman[48547]: 2024-05-15 15:38:48.1694278 +0000 UTC m=+0.017878057 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-05-15T15:38:48.285 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 podman[48547]: 2024-05-15 15:38:48.283019248 +0000 UTC m=+0.131469527 container create 2c6976b8c648e4534859d70e55ba5e605030af13a1e34bd6177d65e6a1e8523c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:38:48.603 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 podman[48547]: 2024-05-15 15:38:48.501889694 +0000 UTC m=+0.350339933 container init 2c6976b8c648e4534859d70e55ba5e605030af13a1e34bd6177d65e6a1e8523c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:38:48.603 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 podman[48547]: 2024-05-15 15:38:48.505006983 +0000 UTC m=+0.353457229 container start 2c6976b8c648e4534859d70e55ba5e605030af13a1e34bd6177d65e6a1e8523c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:38:48.603 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.508Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-05-15T15:38:48.603 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.508Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-05-15T15:38:48.603 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.509Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-05-15T15:38:48.603 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.509Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-15T15:38:48.603 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-05-15T15:38:48.604 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=arp 2024-05-15T15:38:48.604 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=bcache 2024-05-15T15:38:48.604 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=bonding 2024-05-15T15:38:48.604 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=btrfs 2024-05-15T15:38:48.604 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=conntrack 2024-05-15T15:38:48.604 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=cpu 2024-05-15T15:38:48.604 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-05-15T15:38:48.604 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=diskstats 2024-05-15T15:38:48.604 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=dmi 2024-05-15T15:38:48.604 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=edac 2024-05-15T15:38:48.605 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=entropy 2024-05-15T15:38:48.605 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-05-15T15:38:48.605 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=filefd 2024-05-15T15:38:48.605 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=filesystem 2024-05-15T15:38:48.605 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=hwmon 2024-05-15T15:38:48.605 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=infiniband 2024-05-15T15:38:48.605 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=ipvs 2024-05-15T15:38:48.605 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=loadavg 2024-05-15T15:38:48.605 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=mdadm 2024-05-15T15:38:48.605 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=meminfo 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=netclass 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=netdev 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=netstat 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=nfs 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=nfsd 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=nvme 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=os 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=pressure 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=rapl 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=schedstat 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=sockstat 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=softnet 2024-05-15T15:38:48.606 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=stat 2024-05-15T15:38:48.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=tapestats 2024-05-15T15:38:48.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=textfile 2024-05-15T15:38:48.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-05-15T15:38:48.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=time 2024-05-15T15:38:48.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-05-15T15:38:48.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=uname 2024-05-15T15:38:48.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=vmstat 2024-05-15T15:38:48.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=xfs 2024-05-15T15:38:48.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:115 level=info collector=zfs 2024-05-15T15:38:48.610 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.510Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-05-15T15:38:48.611 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[48561]: ts=2024-05-15T15:38:48.511Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-05-15T15:38:48.611 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 bash[48547]: 2c6976b8c648e4534859d70e55ba5e605030af13a1e34bd6177d65e6a1e8523c 2024-05-15T15:38:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-mon[25205]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1300788763"}]': finished 2024-05-15T15:38:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-mon[25205]: osdmap e102: 8 total, 8 up, 8 in 2024-05-15T15:38:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2546485348' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1094161927"}]: dispatch 2024-05-15T15:38:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:38:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:48 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:48.880 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:38:48 smithi053 systemd[1]: Started Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:38:49.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[24430]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1300788763"}]': finished 2024-05-15T15:38:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[24430]: osdmap e102: 8 total, 8 up, 8 in 2024-05-15T15:38:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2546485348' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1094161927"}]: dispatch 2024-05-15T15:38:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:38:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:49.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[29127]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1300788763"}]': finished 2024-05-15T15:38:49.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[29127]: osdmap e102: 8 total, 8 up, 8 in 2024-05-15T15:38:49.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2546485348' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1094161927"}]: dispatch 2024-05-15T15:38:49.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:38:49.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:49.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:48 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:50.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[24430]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T15:38:50.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[24430]: Reconfiguring grafana.a (dependencies changed)... 2024-05-15T15:38:50.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[24430]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-15T15:38:50.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2546485348' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1094161927"}]': finished 2024-05-15T15:38:50.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[24430]: osdmap e103: 8 total, 8 up, 8 in 2024-05-15T15:38:50.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:50.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[24430]: Reconfiguring daemon grafana.a on smithi053 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[29127]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[29127]: Reconfiguring grafana.a (dependencies changed)... 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[29127]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2546485348' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1094161927"}]': finished 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[29127]: osdmap e103: 8 total, 8 up, 8 in 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:38:50.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:38:50.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:49 smithi043 ceph-mon[29127]: Reconfiguring daemon grafana.a on smithi053 2024-05-15T15:38:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:49 smithi053 ceph-mon[25205]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T15:38:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:49 smithi053 ceph-mon[25205]: Reconfiguring grafana.a (dependencies changed)... 2024-05-15T15:38:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:49 smithi053 ceph-mon[25205]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-15T15:38:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:49 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2546485348' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1094161927"}]': finished 2024-05-15T15:38:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:49 smithi053 ceph-mon[25205]: osdmap e103: 8 total, 8 up, 8 in 2024-05-15T15:38:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:49 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:49 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:50.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:49 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:38:50.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:49 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-15T15:38:50.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:49 smithi053 ceph-mon[25205]: Reconfiguring daemon grafana.a on smithi053 2024-05-15T15:38:51.563 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:51 smithi053 ceph-mon[25205]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:38:51.563 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:51 smithi053 systemd[1]: Stopping Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:38:51.563 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:51 smithi053 bash[48732]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana.a" found: no such container 2024-05-15T15:38:51.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:51 smithi043 ceph-mon[24430]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:38:51.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:51 smithi043 ceph-mon[29127]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:38:51.855 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:51 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[41161]: t=2024-05-15T15:38:51+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-05-15T15:38:51.855 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:51 smithi053 podman[48742]: 2024-05-15 15:38:51.571796613 +0000 UTC m=+0.110617481 container died 3e2dc1d42275ddc9ca3498b06ba358d44716633b9273b58a1e1444f081162246 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, release=236.1648460182, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2) 2024-05-15T15:38:52.130 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:51 smithi053 podman[48742]: 2024-05-15 15:38:51.927241135 +0000 UTC m=+0.466061982 container cleanup 3e2dc1d42275ddc9ca3498b06ba358d44716633b9273b58a1e1444f081162246 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., name=ubi8, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, distribution-scope=public, version=8.5, architecture=x86_64, maintainer=Paul Cuzner , vcs-type=git, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8) 2024-05-15T15:38:52.130 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:51 smithi053 bash[48742]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a 2024-05-15T15:38:52.130 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:51 smithi053 bash[48768]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana.a" found: no such container 2024-05-15T15:38:52.489 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:52 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:52.489 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 podman[48755]: 2024-05-15 15:38:52.228401523 +0000 UTC m=+0.653094018 container remove 3e2dc1d42275ddc9ca3498b06ba358d44716633b9273b58a1e1444f081162246 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vendor=Red Hat, Inc., description=Ceph Grafana Container, distribution-scope=public, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, io.openshift.tags=base rhel8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2) 2024-05-15T15:38:52.489 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-15T15:38:52.489 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 systemd[1]: Stopped Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:38:52.489 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@grafana.a.service: Consumed 3.362s CPU time. 2024-05-15T15:38:52.489 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 systemd[1]: Starting Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:38:52.489 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 podman[48831]: 2024-05-15 15:38:52.380458004 +0000 UTC m=+0.017438941 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-05-15T15:38:52.490 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 podman[48831]: 2024-05-15 15:38:52.487905365 +0000 UTC m=+0.124886349 container create 3fece04315b10c927655a1d132a99bc43c1663b10c3257d16d07b5963cee873a (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, io.openshift.tags=base rhel8, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, distribution-scope=public, version=8.5, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, name=ubi8, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762) 2024-05-15T15:38:52.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:52 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:52.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:52 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:38:52.743 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 podman[48831]: 2024-05-15 15:38:52.697422374 +0000 UTC m=+0.334403314 container init 3fece04315b10c927655a1d132a99bc43c1663b10c3257d16d07b5963cee873a (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-03-28T10:36:18.413762, name=ubi8, version=8.5, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, vcs-type=git, architecture=x86_64, distribution-scope=public, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-05-15T15:38:52.743 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 podman[48831]: 2024-05-15 15:38:52.700370718 +0000 UTC m=+0.337351655 container start 3fece04315b10c927655a1d132a99bc43c1663b10c3257d16d07b5963cee873a (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, release=236.1648460182, io.buildah.version=1.24.2, com.redhat.component=ubi8-container, architecture=x86_64, io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, distribution-scope=public) 2024-05-15T15:38:52.743 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-05-15T15:38:52.743 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-05-15T15:38:52.743 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-05-15T15:38:52.743 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-05-15T15:38:52.743 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-15T15:38:52.743 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-15T15:38:52.743 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-15T15:38:52.743 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-15T15:38:52.744 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-05-15T15:38:52.744 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-05-15T15:38:52.744 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-05-15T15:38:52.744 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-05-15T15:38:52.744 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-05-15T15:38:52.744 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="App mode production" logger=settings 2024-05-15T15:38:52.744 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-05-15T15:38:52.744 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-15T15:38:52.744 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-05-15T15:38:52.998 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:52] ENGINE Bus STOPPING 2024-05-15T15:38:52.999 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-15T15:38:52.999 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:52] ENGINE Bus STOPPED 2024-05-15T15:38:52.999 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:52] ENGINE Bus STARTING 2024-05-15T15:38:52.999 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:52] ENGINE Serving on http://:::9283 2024-05-15T15:38:52.999 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:52] ENGINE Bus STARTED 2024-05-15T15:38:52.999 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:52] ENGINE Bus STOPPING 2024-05-15T15:38:52.999 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-15T15:38:52.999 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:52] ENGINE Bus STOPPED 2024-05-15T15:38:52.999 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:52] ENGINE Bus STARTING 2024-05-15T15:38:52.999 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=545.029µs 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 bash[48831]: 3fece04315b10c927655a1d132a99bc43c1663b10c3257d16d07b5963cee873a 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 systemd[1]: Started Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-05-15T15:38:53.000 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:38:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:38:52+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-05-15T15:38:53.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:53] ENGINE Serving on http://:::9283 2024-05-15T15:38:53.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:53] ENGINE Bus STARTED 2024-05-15T15:38:53.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:53] ENGINE Bus STOPPING 2024-05-15T15:38:53.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-15T15:38:53.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:53] ENGINE Bus STOPPED 2024-05-15T15:38:53.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:53] ENGINE Bus STARTING 2024-05-15T15:38:53.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:53] ENGINE Serving on http://:::9283 2024-05-15T15:38:53.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:38:53] ENGINE Bus STARTED 2024-05-15T15:38:53.757 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:38:54.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:54.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:38:54.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:38:54.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi043.front.sepia.ceph.com:9093"}]: dispatch 2024-05-15T15:38:54.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi043.front.sepia.ceph.com:9093"}]: dispatch 2024-05-15T15:38:54.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:38:54.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:38:54.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: Adding iSCSI gateway http://:@172.21.15.43:5000 to Dashboard 2024-05-15T15:38:54.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:38:54.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:38:54.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:38:54.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:38:54.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:38:54.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:38:54.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi053.front.sepia.ceph.com:9095"}]: dispatch 2024-05-15T15:38:54.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi053.front.sepia.ceph.com:9095"}]: dispatch 2024-05-15T15:38:54.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:38:54.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:38:54.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi053.front.sepia.ceph.com:3000"}]: dispatch 2024-05-15T15:38:54.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi053.front.sepia.ceph.com:3000"}]: dispatch 2024-05-15T15:38:54.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:38:54.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:54.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:38:54.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:38:54.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi043.front.sepia.ceph.com:9093"}]: dispatch 2024-05-15T15:38:54.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi043.front.sepia.ceph.com:9093"}]: dispatch 2024-05-15T15:38:54.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:38:54.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:38:54.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: Adding iSCSI gateway http://:@172.21.15.43:5000 to Dashboard 2024-05-15T15:38:54.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:38:54.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:38:54.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:38:54.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:38:54.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.094 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:38:54.094 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:38:54.094 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi053.front.sepia.ceph.com:9095"}]: dispatch 2024-05-15T15:38:54.094 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi053.front.sepia.ceph.com:9095"}]: dispatch 2024-05-15T15:38:54.094 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.094 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:38:54.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:38:54.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi053.front.sepia.ceph.com:3000"}]: dispatch 2024-05-15T15:38:54.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi053.front.sepia.ceph.com:3000"}]: dispatch 2024-05-15T15:38:54.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:53 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:38:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:38:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:38:54.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:38:54.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi043.front.sepia.ceph.com:9093"}]: dispatch 2024-05-15T15:38:54.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi043.front.sepia.ceph.com:9093"}]: dispatch 2024-05-15T15:38:54.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:38:54.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T15:38:54.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: Adding iSCSI gateway http://:@172.21.15.43:5000 to Dashboard 2024-05-15T15:38:54.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:38:54.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T15:38:54.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:38:54.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi043"}]: dispatch 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi053.front.sepia.ceph.com:9095"}]: dispatch 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi053.front.sepia.ceph.com:9095"}]: dispatch 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi053.front.sepia.ceph.com:3000"}]: dispatch 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi053.front.sepia.ceph.com:3000"}]: dispatch 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:54.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:53 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:38:54.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:38:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:38:54] "GET /metrics HTTP/1.1" 200 37885 "" "Prometheus/2.33.4" 2024-05-15T15:38:55.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:55 smithi053 ceph-mon[25205]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 657 B/s rd, 0 op/s 2024-05-15T15:38:56.191 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:55 smithi043 ceph-mon[29127]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 657 B/s rd, 0 op/s 2024-05-15T15:38:56.191 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:55 smithi043 ceph-mon[24430]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 657 B/s rd, 0 op/s 2024-05-15T15:38:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:57 smithi053 ceph-mon[25205]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:38:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:57 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:57 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:57 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:57 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:57 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:57 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:38:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:57 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:58.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[24430]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:38:58.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:58.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:58.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:58.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:58.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:58.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:38:58.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:58.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[29127]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:38:58.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:58.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:58.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:58.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:58.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:38:58.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:38:58.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:57 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:38:59.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:38:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:38:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:38:59 smithi043 ceph-mon[24430]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:39:00.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:38:59 smithi043 ceph-mon[29127]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:39:00.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:38:59 smithi053 ceph-mon[25205]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:39:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:01 smithi043 ceph-mon[24430]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:39:01.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:01 smithi043 ceph-mon[29127]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:39:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:01 smithi053 ceph-mon[25205]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:39:02.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:02 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:02.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:02 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:02 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:03.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:03 smithi043 ceph-mon[24430]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:03.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:03 smithi043 ceph-mon[29127]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:03 smithi053 ceph-mon[25205]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:03.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:03.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:04.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:04 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:04.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:04 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:04.602 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:04 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:04.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:39:04 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:39:04] "GET /metrics HTTP/1.1" 200 37882 "" "Prometheus/2.33.4" 2024-05-15T15:39:05.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:05 smithi043 ceph-mon[24430]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:05.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:05 smithi043 ceph-mon[29127]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:05 smithi053 ceph-mon[25205]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:07.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:07 smithi043 ceph-mon[24430]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:07.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:07 smithi043 ceph-mon[29127]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:07 smithi053 ceph-mon[25205]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:09.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:09 smithi043 ceph-mon[24430]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:09 smithi043 ceph-mon[29127]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:09.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:09 smithi053 ceph-mon[25205]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:11 smithi043 ceph-mon[24430]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:11 smithi043 ceph-mon[29127]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:11 smithi053 ceph-mon[25205]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:12.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:12 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:12.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:12 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:12 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:13.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:13 smithi043 ceph-mon[24430]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:13.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:13 smithi043 ceph-mon[29127]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:13 smithi053 ceph-mon[25205]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:13.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:13 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:14.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:39:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:39:14] "GET /metrics HTTP/1.1" 200 37882 "" "Prometheus/2.33.4" 2024-05-15T15:39:15.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:15 smithi043 ceph-mon[24430]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:15 smithi043 ceph-mon[29127]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:15 smithi053 ceph-mon[25205]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:17 smithi043 ceph-mon[24430]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:17 smithi043 ceph-mon[29127]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:17 smithi053 ceph-mon[25205]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:19.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:19.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:19 smithi043 ceph-mon[24430]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:19 smithi043 ceph-mon[29127]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:19 smithi053 ceph-mon[25205]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:21.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:21 smithi043 ceph-mon[24430]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:21 smithi043 ceph-mon[29127]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:21 smithi053 ceph-mon[25205]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:22.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:22 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:22.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:22 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:22.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:22 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:23.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:23 smithi043 ceph-mon[24430]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:23.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:23 smithi043 ceph-mon[29127]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:23.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:23 smithi053 ceph-mon[25205]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:23.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:24.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:39:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:39:24] "GET /metrics HTTP/1.1" 200 37881 "" "Prometheus/2.33.4" 2024-05-15T15:39:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:25 smithi043 ceph-mon[24430]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:25 smithi043 ceph-mon[29127]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:25 smithi053 ceph-mon[25205]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:27.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:27 smithi043 ceph-mon[29127]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:27 smithi043 ceph-mon[24430]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:27 smithi053 ceph-mon[25205]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:29.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:29.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:29 smithi043 ceph-mon[29127]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:29 smithi043 ceph-mon[24430]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:29 smithi053 ceph-mon[25205]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:31.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:31 smithi043 ceph-mon[29127]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:31.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:31 smithi043 ceph-mon[24430]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:31.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:31 smithi053 ceph-mon[25205]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:32.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:32 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:32.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:32 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:32.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:32 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:33.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:33 smithi043 ceph-mon[24430]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:33.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:33 smithi043 ceph-mon[29127]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:33.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:33 smithi053 ceph-mon[25205]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:33.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:34.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:34 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:34.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:34 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:34.603 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:34 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:34.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:39:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:39:34] "GET /metrics HTTP/1.1" 200 37879 "" "Prometheus/2.33.4" 2024-05-15T15:39:35.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:35 smithi043 ceph-mon[24430]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:35 smithi043 ceph-mon[29127]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:35.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:35 smithi053 ceph-mon[25205]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:37.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:37 smithi043 ceph-mon[24430]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:37 smithi043 ceph-mon[29127]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:37 smithi053 ceph-mon[25205]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:39.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:39.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:39 smithi043 ceph-mon[24430]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:39 smithi043 ceph-mon[29127]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:39 smithi053 ceph-mon[25205]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:41.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:41 smithi043 ceph-mon[24430]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:41.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:41 smithi043 ceph-mon[29127]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:41 smithi053 ceph-mon[25205]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:42.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:42 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:42.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:42 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:42 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:43.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:43 smithi043 ceph-mon[24430]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:43.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:43 smithi043 ceph-mon[29127]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:43 smithi053 ceph-mon[25205]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:43.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:43.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:44.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:39:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:39:44] "GET /metrics HTTP/1.1" 200 37879 "" "Prometheus/2.33.4" 2024-05-15T15:39:45.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:45 smithi043 ceph-mon[24430]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:45 smithi043 ceph-mon[29127]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:45 smithi053 ceph-mon[25205]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:47.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:47 smithi043 ceph-mon[24430]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:47 smithi043 ceph-mon[29127]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:47 smithi053 ceph-mon[25205]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:49.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:49.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:49 smithi043 ceph-mon[24430]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:49.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:49 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:49 smithi043 ceph-mon[29127]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:49 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:49 smithi053 ceph-mon[25205]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:49 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:39:51.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:51 smithi043 ceph-mon[24430]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:51.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:51 smithi043 ceph-mon[29127]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:51 smithi053 ceph-mon[25205]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:52.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:52 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:52.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:52 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:52.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:52 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:39:53.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:53 smithi043 ceph-mon[24430]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:53.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:53 smithi043 ceph-mon[29127]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:53.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:53 smithi053 ceph-mon[25205]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:53.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:53.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:54.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:39:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:39:54] "GET /metrics HTTP/1.1" 200 37876 "" "Prometheus/2.33.4" 2024-05-15T15:39:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:55 smithi043 ceph-mon[24430]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:55 smithi043 ceph-mon[29127]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:55 smithi053 ceph-mon[25205]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:39:57.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:57 smithi043 ceph-mon[24430]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:57 smithi043 ceph-mon[29127]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:57 smithi053 ceph-mon[25205]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:58.660 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:58 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:39:58.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:58 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:39:58.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:58 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:39:59.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:39:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:39:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:39:59.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:39:59 smithi043 ceph-mon[24430]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:39:59 smithi043 ceph-mon[29127]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:39:59.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:39:59 smithi053 ceph-mon[25205]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:00 smithi043 ceph-mon[24430]: overall HEALTH_OK 2024-05-15T15:40:00.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:00 smithi043 ceph-mon[29127]: overall HEALTH_OK 2024-05-15T15:40:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:00 smithi053 ceph-mon[25205]: overall HEALTH_OK 2024-05-15T15:40:01.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:01 smithi043 ceph-mon[24430]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:01.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:01 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:01.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:01 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:01 smithi043 ceph-mon[29127]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:01 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:01 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:01 smithi053 ceph-mon[25205]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:01 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:01 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:03.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:03 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:03.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:03 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:03.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:03 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:03.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:03 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:40:03.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:03 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:40:03.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:03 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:03.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:40:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:40:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:03.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:03.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:03.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:03.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:40:03.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:40:03.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:03 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:40:03.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:04.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:04 smithi053 ceph-mon[25205]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:04.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:04 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:04.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:04 smithi043 ceph-mon[24430]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:04.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:04 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:04.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:04 smithi043 ceph-mon[29127]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:04.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:04 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:04.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:40:04 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:40:04] "GET /metrics HTTP/1.1" 200 37875 "" "Prometheus/2.33.4" 2024-05-15T15:40:05.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:05 smithi053 ceph-mon[25205]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:05.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:05 smithi043 ceph-mon[24430]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:05.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:05 smithi043 ceph-mon[29127]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:07.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:07 smithi043 ceph-mon[24430]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:07.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:07 smithi043 ceph-mon[29127]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:07 smithi053 ceph-mon[25205]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:09.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:09.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:09 smithi043 ceph-mon[24430]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:09 smithi043 ceph-mon[29127]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:09.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:09 smithi053 ceph-mon[25205]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:11 smithi043 ceph-mon[24430]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:11 smithi043 ceph-mon[29127]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:11 smithi053 ceph-mon[25205]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:12.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:12 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:12.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:12 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:12 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:13 smithi053 ceph-mon[25205]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:13.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:13 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:13.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:13 smithi043 ceph-mon[24430]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:13 smithi043 ceph-mon[29127]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:14.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:40:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:40:14] "GET /metrics HTTP/1.1" 200 37875 "" "Prometheus/2.33.4" 2024-05-15T15:40:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:15 smithi043 ceph-mon[24430]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:15.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:15 smithi043 ceph-mon[29127]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:15 smithi053 ceph-mon[25205]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:17.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:17 smithi043 ceph-mon[24430]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:17 smithi043 ceph-mon[29127]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:17 smithi053 ceph-mon[25205]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:19.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:19.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:19 smithi043 ceph-mon[24430]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:19 smithi043 ceph-mon[29127]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:19 smithi053 ceph-mon[25205]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:21.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:21 smithi043 ceph-mon[24430]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:21 smithi043 ceph-mon[29127]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:21 smithi053 ceph-mon[25205]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:22.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:22 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:22.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:22 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:22.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:22 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:23.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:23 smithi053 ceph-mon[25205]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:23.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:23.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:23 smithi043 ceph-mon[24430]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:23 smithi043 ceph-mon[29127]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:24.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:40:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:40:24] "GET /metrics HTTP/1.1" 200 37874 "" "Prometheus/2.33.4" 2024-05-15T15:40:25.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:25 smithi043 ceph-mon[24430]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:25 smithi043 ceph-mon[29127]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:25 smithi053 ceph-mon[25205]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:27 smithi043 ceph-mon[24430]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:27.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:27 smithi043 ceph-mon[29127]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:27 smithi053 ceph-mon[25205]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:29.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:29.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:29 smithi043 ceph-mon[24430]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:29 smithi043 ceph-mon[29127]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:29 smithi053 ceph-mon[25205]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:31.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:31 smithi053 ceph-mon[25205]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:31.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:31 smithi043 ceph-mon[24430]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:31.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:31 smithi043 ceph-mon[29127]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:32.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:32 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:32.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:32 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:32.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:32 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:33.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:33 smithi053 ceph-mon[25205]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:33.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:33.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:33 smithi043 ceph-mon[24430]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:33 smithi043 ceph-mon[29127]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:34.603 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:34 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:34 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:34.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:34 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:34.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:40:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:40:34] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-15T15:40:35.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:35 smithi053 ceph-mon[25205]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:35 smithi043 ceph-mon[24430]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:35 smithi043 ceph-mon[29127]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:37.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:37 smithi043 ceph-mon[24430]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:37 smithi043 ceph-mon[29127]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:37 smithi053 ceph-mon[25205]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:39.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:39.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:39 smithi043 ceph-mon[24430]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:39 smithi043 ceph-mon[29127]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:39 smithi053 ceph-mon[25205]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:41 smithi053 ceph-mon[25205]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:41.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:41 smithi043 ceph-mon[24430]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:41 smithi043 ceph-mon[29127]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:42 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:42 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:42 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:43 smithi053 ceph-mon[25205]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:43.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:43 smithi043 ceph-mon[24430]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:43 smithi043 ceph-mon[29127]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:44.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:40:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:40:44] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-15T15:40:45.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:45 smithi043 ceph-mon[29127]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:45 smithi043 ceph-mon[24430]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:45 smithi053 ceph-mon[25205]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:47.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:47 smithi043 ceph-mon[24430]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:47 smithi043 ceph-mon[29127]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:47 smithi053 ceph-mon[25205]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:49.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:49 smithi043 ceph-mon[29127]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:49 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:49 smithi043 ceph-mon[24430]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:49 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:49 smithi053 ceph-mon[25205]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:49 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:40:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:51 smithi053 ceph-mon[25205]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:51.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:51 smithi043 ceph-mon[24430]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:51 smithi043 ceph-mon[29127]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:52.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:52 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:52.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:52 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:52.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:52 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:40:53.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:53 smithi053 ceph-mon[25205]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:53.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:53.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:53 smithi043 ceph-mon[24430]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:53 smithi043 ceph-mon[29127]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:54.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:40:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:40:54] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-05-15T15:40:55.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:55 smithi043 ceph-mon[24430]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:55 smithi043 ceph-mon[29127]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:55 smithi053 ceph-mon[25205]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:40:57.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:57 smithi043 ceph-mon[24430]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:57 smithi043 ceph-mon[29127]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:57 smithi053 ceph-mon[25205]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:40:59.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:40:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:40:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:40:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:40:59 smithi053 ceph-mon[25205]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:40:59 smithi043 ceph-mon[24430]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:00.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:40:59 smithi043 ceph-mon[29127]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:01 smithi053 ceph-mon[25205]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:01.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:01 smithi043 ceph-mon[24430]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:01 smithi043 ceph-mon[29127]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:02 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:02 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:02 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:03 smithi053 ceph-mon[25205]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:03 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:41:03.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:03.745 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:03 smithi043 ceph-mon[29127]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:03.745 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:03 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:41:03.745 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:03 smithi043 ceph-mon[24430]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:03.745 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:03 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:41:04.603 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:04 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:04.745 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:04 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:04.746 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:04 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:04.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:41:04 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:41:04] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.33.4" 2024-05-15T15:41:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:05 smithi053 ceph-mon[25205]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:05.677 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:05 smithi043 ceph-mon[24430]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:05.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:05 smithi043 ceph-mon[29127]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[24430]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[29127]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:08.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:41:08.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:07 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:07 smithi053 ceph-mon[25205]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:07 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:07 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:07 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:07 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:07 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:07 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:41:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:07 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:09.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:09 smithi043 ceph-mon[24430]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:09 smithi043 ceph-mon[29127]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:09 smithi053 ceph-mon[25205]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:11 smithi053 ceph-mon[25205]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:11.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:11 smithi043 ceph-mon[24430]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:11 smithi043 ceph-mon[29127]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:12 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:12.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:12 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:12.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:12 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:13 smithi053 ceph-mon[25205]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:13.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:13 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:13.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:13 smithi043 ceph-mon[24430]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:13 smithi043 ceph-mon[29127]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:14.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:41:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:41:14] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.33.4" 2024-05-15T15:41:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:15 smithi053 ceph-mon[25205]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:16.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:15 smithi043 ceph-mon[24430]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:16.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:15 smithi043 ceph-mon[29127]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:17 smithi053 ceph-mon[25205]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:18.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:17 smithi043 ceph-mon[24430]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:18.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:17 smithi043 ceph-mon[29127]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:19.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:19 smithi053 ceph-mon[25205]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:19 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:20.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:19 smithi043 ceph-mon[24430]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:20.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:19 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:20.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:19 smithi043 ceph-mon[29127]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:20.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:19 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:21.439 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:21 smithi043 ceph-mon[24430]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:21.439 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:21 smithi043 ceph-mon[29127]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:21.470 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-15T15:41:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:21 smithi053 ceph-mon[25205]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:22.538 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:22 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:22.538 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:22 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:22.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:22 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:23.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:23 smithi043 ceph-mon[24430]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:23.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:23 smithi043 ceph-mon[29127]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:23.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:23 smithi053 ceph-mon[25205]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:23.637 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:23.980 INFO:teuthology.orchestra.run.smithi043.stdout:Scheduled to redeploy mgr.y on host 'smithi043' 2024-05-15T15:41:24.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1393274994' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-15T15:41:24.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[24430]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:41:24.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:24.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[24430]: Schedule redeploy daemon mgr.y 2024-05-15T15:41:24.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:24.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:24.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:41:24.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1393274994' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-15T15:41:24.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[29127]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:41:24.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:24.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[29127]: Schedule redeploy daemon mgr.y 2024-05-15T15:41:24.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:24.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:24.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:24 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:41:24.617 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps --refresh' 2024-05-15T15:41:24.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:41:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:41:24] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.33.4" 2024-05-15T15:41:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:24 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1393274994' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-15T15:41:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:24 smithi053 ceph-mon[25205]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:41:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:24 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:24 smithi053 ceph-mon[25205]: Schedule redeploy daemon mgr.y 2024-05-15T15:41:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:24 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:24 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:24 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:41:25.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:25 smithi043 ceph-mon[24430]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:25.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:25 smithi043 ceph-mon[29127]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:25 smithi053 ceph-mon[25205]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:26.713 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (2m) 19s ago 10m 24.1M - 0.23.0 ba2b418f427c dd5a5b0f3fe5 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (2m) 19s ago 10m 45.0M - 8.3.5 dad864ee21e9 3fece04315b1 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 running (2m) 19s ago 9m 51.0M - 3.5 e1d6a67b021e daca6b662f1d 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283 running (5m) 19s ago 15m 514M - 19.0.0-2576-gde5210d5 48bc18eb22d8 cfeda1bd207c 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:9283 running (16m) 19s ago 16m 412M - 17.2.0 e1d6a67b021e faaa138d25ee 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (16m) 19s ago 16m 57.8M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (15m) 19s ago 15m 51.0M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (15m) 19s ago 15m 47.8M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (2m) 19s ago 11m 16.7M - 1.3.1 1dbe0e931976 083a05fd3833 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (2m) 19s ago 11m 16.8M - 1.3.1 1dbe0e931976 2c6976b8c648 2024-05-15T15:41:26.714 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (14m) 19s ago 14m 51.6M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:41:26.715 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (14m) 19s ago 14m 55.0M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:41:26.715 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (14m) 19s ago 14m 48.2M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:41:26.715 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (13m) 19s ago 13m 50.2M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:41:26.715 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (13m) 19s ago 13m 50.3M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:41:26.715 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (12m) 19s ago 12m 49.1M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:41:26.715 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (12m) 19s ago 12m 51.5M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:41:26.715 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (12m) 19s ago 12m 53.4M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:41:26.715 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (2m) 19s ago 11m 68.4M - 2.33.4 514e6a882f6e e8019f2a6456 2024-05-15T15:41:26.715 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (9m) 19s ago 9m 72.5M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:41:26.715 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (9m) 19s ago 9m 71.3M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:41:27.563 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'sleep 180' 2024-05-15T15:41:27.855 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:27 smithi043 ceph-mon[24430]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:27.855 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:27 smithi043 ceph-mon[24430]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:41:27.855 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:27 smithi043 ceph-mon[29127]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:27.855 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:27 smithi043 ceph-mon[29127]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:41:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:27 smithi053 ceph-mon[25205]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:27 smithi053 ceph-mon[25205]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:41:29.061 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:29.278 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:29 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:29.279 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:29 smithi043 ceph-mon[24430]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:29.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:29 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:29.279 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:29 smithi043 ceph-mon[29127]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:29 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:29 smithi053 ceph-mon[25205]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:31.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:31 smithi053 ceph-mon[25205]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:31.734 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:31 smithi043 ceph-mon[24430]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:31.734 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:31 smithi043 ceph-mon[29127]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:32.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:32 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:32.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:32 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:32.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:32 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:33.756 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.756 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.756 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:33.756 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.756 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:33.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:41:33.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:41:33.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:41:33.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:41:33.758 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:33.758 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[29127]: Deploying daemon mgr.y on smithi043 2024-05-15T15:41:33.758 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.758 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.759 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:33.759 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.759 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.759 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:33.759 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:41:33.759 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.760 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:41:33.760 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:41:33.760 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:41:33.760 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:33.760 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:33 smithi043 ceph-mon[24430]: Deploying daemon mgr.y on smithi043 2024-05-15T15:41:33.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:41:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:33.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:41:33.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:41:33.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:41:33.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:33.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:33 smithi053 ceph-mon[25205]: Deploying daemon mgr.y on smithi043 2024-05-15T15:41:34.744 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:34 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:34.744 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:34 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:34 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:34.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:41:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:41:34] "GET /metrics HTTP/1.1" 200 37881 "" "Prometheus/2.33.4" 2024-05-15T15:41:35.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:34 smithi043 systemd[1]: Stopping Ceph mgr.y for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:41:35.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:35 smithi043 ceph-mon[24430]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:35.568 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:35 smithi043 ceph-mon[29127]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:35.568 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:35 smithi043 podman[61999]: 2024-05-15 15:41:35.254483495 +0000 UTC m=+0.152995652 container died faaa138d25eefb4a83dee6dfc6e9cc184466848f46aa7c4cf0cac44c269820f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.openshift.tags=base centos centos-stream, name=centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD) 2024-05-15T15:41:35.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:35 smithi043 podman[61999]: 2024-05-15 15:41:35.656913746 +0000 UTC m=+0.555425895 container cleanup faaa138d25eefb4a83dee6dfc6e9cc184466848f46aa7c4cf0cac44c269820f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.expose-services=, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, ceph=True, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-15T15:41:35.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:35 smithi043 bash[61999]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y 2024-05-15T15:41:35.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:35 smithi053 ceph-mon[25205]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:36.255 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:35 smithi043 podman[62014]: 2024-05-15 15:41:35.892083089 +0000 UTC m=+0.640565637 container remove faaa138d25eefb4a83dee6dfc6e9cc184466848f46aa7c4cf0cac44c269820f3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, RELEASE=HEAD, io.openshift.expose-services=, vcs-type=git, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-15T15:41:36.255 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:35 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-05-15T15:41:36.255 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-05-15T15:41:36.255 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 systemd[1]: Stopped Ceph mgr.y for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:41:36.255 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.y.service: Consumed 1min 3.267s CPU time. 2024-05-15T15:41:36.585 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 systemd[1]: Starting Ceph mgr.y for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:41:36.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 podman[62141]: 2024-05-15 15:41:36.375259914 +0000 UTC m=+0.018092930 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:41:36.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 podman[62141]: 2024-05-15 15:41:36.485412059 +0000 UTC m=+0.128245070 container create 8de8e1ec414e1e5d6059aa72e5778400f04f8d1d21c9f166c6badd61db01d886 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-05-15T15:41:36.892 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 podman[62141]: 2024-05-15 15:41:36.713006114 +0000 UTC m=+0.355839122 container init 8de8e1ec414e1e5d6059aa72e5778400f04f8d1d21c9f166c6badd61db01d886 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-15T15:41:36.892 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 podman[62141]: 2024-05-15 15:41:36.71602072 +0000 UTC m=+0.358853738 container start 8de8e1ec414e1e5d6059aa72e5778400f04f8d1d21c9f166c6badd61db01d886 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=) 2024-05-15T15:41:36.892 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 ceph-mgr[62161]: -- 172.21.15.43:0/4088695435 <== mon.0 v2:172.21.15.43:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559bfc8e65a0 con 0x559bfc8bac00 2024-05-15T15:41:36.892 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 bash[62141]: 8de8e1ec414e1e5d6059aa72e5778400f04f8d1d21c9f166c6badd61db01d886 2024-05-15T15:41:36.892 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 systemd[1]: Started Ceph mgr.y for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:41:37.204 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:36.889+0000 7f69ec23e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:41:37.204 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:37.090+0000 7f69ec23e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:41:37.204 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:37.202+0000 7f69ec23e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:41:37.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:37.554+0000 7f69ec23e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:41:37.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:37.648+0000 7f69ec23e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:41:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:37 smithi053 ceph-mon[25205]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:37 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:37 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:37 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:37 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:37 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:41:38.335 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[29127]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:38.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:41:38.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[24430]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:38.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:38.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:37 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:41:39.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:39.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:38 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:38.970+0000 7f69ec23e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:41:39.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:39.171+0000 7f69ec23e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:41:39.649 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:39.358+0000 7f69ec23e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:41:40.004 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:39 smithi043 ceph-mon[24430]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:40.005 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:39 smithi043 ceph-mon[29127]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:40.005 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:39.674+0000 7f69ec23e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:41:40.005 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:39.773+0000 7f69ec23e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:41:40.273 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:40.002+0000 7f69ec23e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:41:40.273 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:40.210+0000 7f69ec23e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:41:40.342 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:39 smithi053 ceph-mon[25205]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:41.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:40.870+0000 7f69ec23e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:41:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:41 smithi053 ceph-mon[25205]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-15T15:41:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:41 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:41 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:41 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:41 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:41:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:41 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:41.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[24430]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-15T15:41:41.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:41.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:41.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:41.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:41:41.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:41.667 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:41 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:41.337+0000 7f69ec23e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:41:41.667 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:41 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:41.448+0000 7f69ec23e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:41:41.668 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:41 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:41.664+0000 7f69ec23e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-15T15:41:41.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[29127]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-15T15:41:41.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:41.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:41.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:41:41.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:41:41.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:41 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:41:42.324 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:42.223+0000 7f69ec23e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:41:42.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:42 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:42.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:42 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:42.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:42.322+0000 7f69ec23e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:41:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:42 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:42.962 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:42.586+0000 7f69ec23e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:41:42.962 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:42.730+0000 7f69ec23e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:41:42.962 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:42.834+0000 7f69ec23e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:41:42.962 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:42.959+0000 7f69ec23e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:41:43.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:43 smithi043 ceph-mon[24430]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-15T15:41:43.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:43 smithi043 ceph-mon[29127]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-15T15:41:43.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:43.279+0000 7f69ec23e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:41:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:43 smithi053 ceph-mon[25205]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-15T15:41:43.631 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:44.279 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:43.932+0000 7f69ec23e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:41:44.279 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:44.168+0000 7f69ec23e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:41:44.279 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:41:44] ENGINE Bus STARTING 2024-05-15T15:41:44.279 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CherryPy Checker: 2024-05-15T15:41:44.279 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: The Application mounted at '' has an empty config. 2024-05-15T15:41:44.279 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:41:44.280 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:41:44] ENGINE Serving on http://:::9283 2024-05-15T15:41:44.280 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:41:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:41:44] ENGINE Bus STARTED 2024-05-15T15:41:44.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[24430]: Standby manager daemon y restarted 2024-05-15T15:41:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[24430]: Standby manager daemon y started 2024-05-15T15:41:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-15T15:41:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:41:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-15T15:41:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:41:44.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[29127]: Standby manager daemon y restarted 2024-05-15T15:41:44.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[29127]: Standby manager daemon y started 2024-05-15T15:41:44.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-15T15:41:44.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:41:44.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-15T15:41:44.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:44 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:41:44.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:41:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:41:44] "GET /metrics HTTP/1.1" 200 37881 "" "Prometheus/2.33.4" 2024-05-15T15:41:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:44 smithi053 ceph-mon[25205]: Standby manager daemon y restarted 2024-05-15T15:41:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:44 smithi053 ceph-mon[25205]: Standby manager daemon y started 2024-05-15T15:41:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:44 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-15T15:41:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:44 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:41:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:44 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-15T15:41:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:44 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:41:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:45 smithi053 ceph-mon[25205]: mgrmap e27: x(active, since 3m), standbys: y 2024-05-15T15:41:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:45 smithi053 ceph-mon[25205]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-15T15:41:45.748 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:45 smithi043 ceph-mon[24430]: mgrmap e27: x(active, since 3m), standbys: y 2024-05-15T15:41:45.749 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:45 smithi043 ceph-mon[24430]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-15T15:41:45.749 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:45 smithi043 ceph-mon[29127]: mgrmap e27: x(active, since 3m), standbys: y 2024-05-15T15:41:45.749 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:45 smithi043 ceph-mon[29127]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-15T15:41:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:47 smithi053 ceph-mon[25205]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:48.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:47 smithi043 ceph-mon[29127]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:48.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:47 smithi043 ceph-mon[24430]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:48 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:48.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:49.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:48 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:49.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:48 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:41:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:49 smithi053 ceph-mon[25205]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:50.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:49 smithi043 ceph-mon[24430]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:49 smithi043 ceph-mon[29127]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:51 smithi053 ceph-mon[25205]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:51.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:51 smithi043 ceph-mon[24430]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:51 smithi043 ceph-mon[29127]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:41:52.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:52 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:52.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:52 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:52.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:52 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:41:53.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:53 smithi053 ceph-mon[25205]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:53.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:53 smithi043 ceph-mon[24430]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:53 smithi043 ceph-mon[29127]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:54.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:41:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:41:54] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-05-15T15:41:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:55 smithi053 ceph-mon[25205]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:56.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:55 smithi043 ceph-mon[24430]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:56.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:55 smithi043 ceph-mon[29127]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:41:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:57 smithi053 ceph-mon[25205]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:58.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:57 smithi043 ceph-mon[24430]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:58.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:57 smithi043 ceph-mon[29127]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:41:59.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:41:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:41:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:41:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:41:59 smithi053 ceph-mon[25205]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:00.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:41:59 smithi043 ceph-mon[24430]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:00.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:41:59 smithi043 ceph-mon[29127]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:01 smithi053 ceph-mon[25205]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:01.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:01 smithi043 ceph-mon[24430]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:01 smithi043 ceph-mon[29127]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:02 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:02 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:02 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:03 smithi053 ceph-mon[25205]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:42:03.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:03.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:03 smithi043 ceph-mon[29127]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:42:03.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:03 smithi043 ceph-mon[24430]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T15:42:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:04 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:04.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:42:04 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:42:04] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-05-15T15:42:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:04 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:04 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:05 smithi053 ceph-mon[25205]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:05.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:05 smithi043 ceph-mon[24430]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:05 smithi043 ceph-mon[29127]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:07 smithi053 ceph-mon[25205]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:07 smithi043 ceph-mon[24430]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:08.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:07 smithi043 ceph-mon[29127]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:09.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:09 smithi053 ceph-mon[25205]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:09 smithi043 ceph-mon[24430]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:09 smithi043 ceph-mon[29127]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:11 smithi053 ceph-mon[25205]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:11.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:11 smithi043 ceph-mon[24430]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:11 smithi043 ceph-mon[29127]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:12 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:12.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:12 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:12.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:12 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:13 smithi053 ceph-mon[25205]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:13.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:13 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:13.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:13 smithi043 ceph-mon[24430]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:13 smithi043 ceph-mon[29127]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:14.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:42:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:42:14] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-05-15T15:42:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:15 smithi053 ceph-mon[25205]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:15.903 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:15 smithi043 ceph-mon[24430]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:15.903 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:15 smithi043 ceph-mon[29127]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:17 smithi053 ceph-mon[25205]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:18.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:17 smithi043 ceph-mon[24430]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:18.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:17 smithi043 ceph-mon[29127]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:18 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:18.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:19.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:18 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:19.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:18 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:19 smithi053 ceph-mon[25205]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:20.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:19 smithi043 ceph-mon[24430]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:20.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:19 smithi043 ceph-mon[29127]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:21 smithi053 ceph-mon[25205]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:21.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:21 smithi043 ceph-mon[24430]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:21 smithi043 ceph-mon[29127]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:22.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:22 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:22.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:22 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:22 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:23.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:23 smithi053 ceph-mon[25205]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:23.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:23.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:23 smithi043 ceph-mon[24430]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:23 smithi043 ceph-mon[29127]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:24.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:42:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:42:24] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-05-15T15:42:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:25 smithi053 ceph-mon[25205]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:26.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:25 smithi043 ceph-mon[24430]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:26.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:25 smithi043 ceph-mon[29127]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:27 smithi053 ceph-mon[25205]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:28.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:27 smithi043 ceph-mon[24430]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:27 smithi043 ceph-mon[29127]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:29.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:29 smithi053 ceph-mon[25205]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:30.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:29 smithi043 ceph-mon[24430]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:30.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:29 smithi043 ceph-mon[29127]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:31.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:31 smithi053 ceph-mon[25205]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:31.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:31 smithi043 ceph-mon[29127]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:31 smithi043 ceph-mon[24430]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:32.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:32 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:32.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:32 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:32.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:32 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:33.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:33 smithi053 ceph-mon[25205]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:33.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:33.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:33 smithi043 ceph-mon[24430]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:33 smithi043 ceph-mon[29127]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:34.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:34 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:34.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:42:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:42:34] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-05-15T15:42:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:34 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:34.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:34 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:35.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:35 smithi053 ceph-mon[25205]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:35 smithi043 ceph-mon[29127]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:35 smithi043 ceph-mon[24430]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:37 smithi053 ceph-mon[25205]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:38.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:37 smithi043 ceph-mon[29127]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:38.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:37 smithi043 ceph-mon[24430]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:39.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:39 smithi053 ceph-mon[25205]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:39 smithi043 ceph-mon[24430]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:40.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:39 smithi043 ceph-mon[29127]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:41 smithi053 ceph-mon[25205]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:41.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:41 smithi043 ceph-mon[24430]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:41 smithi043 ceph-mon[29127]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:42 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:42:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:42 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:42 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:42:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:42 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:42:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:42 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:42:42.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:42 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:42:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:42 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:42 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:42:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:42 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:42:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:42 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:42:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:42 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:42:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:42 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:42 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:42:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:42 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:42:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:42 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:42:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:43 smithi053 ceph-mon[25205]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:43.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:43.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:43 smithi043 ceph-mon[24430]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:43 smithi043 ceph-mon[29127]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:44.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:42:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:42:44] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-05-15T15:42:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:45 smithi053 ceph-mon[25205]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:46.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:45 smithi043 ceph-mon[29127]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:45 smithi043 ceph-mon[24430]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:47 smithi053 ceph-mon[25205]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:48.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:47 smithi043 ceph-mon[24430]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:48.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:47 smithi043 ceph-mon[29127]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:48 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:48.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:49.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:48 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:49.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:48 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:42:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:49 smithi053 ceph-mon[25205]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:50.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:49 smithi043 ceph-mon[24430]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:49 smithi043 ceph-mon[29127]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:51 smithi053 ceph-mon[25205]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:51.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:51 smithi043 ceph-mon[24430]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:51 smithi043 ceph-mon[29127]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:52.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:52 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:52 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:52.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:52 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:42:53.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:53 smithi043 ceph-mon[24430]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:53 smithi043 ceph-mon[29127]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:53 smithi053 ceph-mon[25205]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:53.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:54.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:42:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:42:54] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-15T15:42:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:55 smithi053 ceph-mon[25205]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:56.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:55 smithi043 ceph-mon[24430]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:56.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:55 smithi043 ceph-mon[29127]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:42:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:57 smithi053 ceph-mon[25205]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:58.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:57 smithi043 ceph-mon[24430]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:58.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:57 smithi043 ceph-mon[29127]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:42:59.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:42:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:42:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:42:59.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:42:59 smithi053 ceph-mon[25205]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:00.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:42:59 smithi043 ceph-mon[24430]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:00.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:42:59 smithi043 ceph-mon[29127]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:01 smithi053 ceph-mon[25205]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:01.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:01 smithi043 ceph-mon[24430]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:01 smithi043 ceph-mon[29127]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:02 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:02 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:02 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:03 smithi053 ceph-mon[25205]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:03.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:43:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:43:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:43:03.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:03 smithi043 ceph-mon[24430]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:03 smithi043 ceph-mon[29127]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:04 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:04 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:04 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:04.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:43:04 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:43:04] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-15T15:43:05.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:05 smithi043 ceph-mon[24430]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:05 smithi043 ceph-mon[29127]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:05 smithi053 ceph-mon[25205]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:07 smithi053 ceph-mon[25205]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:07 smithi043 ceph-mon[24430]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:08.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:07 smithi043 ceph-mon[29127]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:09.043 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:43:08 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:43:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:43:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:09 smithi053 ceph-mon[25205]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:10.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:09 smithi043 ceph-mon[24430]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:09 smithi043 ceph-mon[29127]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:11 smithi053 ceph-mon[25205]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:11.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:11 smithi043 ceph-mon[24430]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:11 smithi043 ceph-mon[29127]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:12 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:12.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:12 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:12.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:12 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:13 smithi053 ceph-mon[25205]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:13.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:43:13 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:43:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:43:13.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:13 smithi043 ceph-mon[29127]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:13 smithi043 ceph-mon[24430]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:14.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:43:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:43:14] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-15T15:43:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:15 smithi053 ceph-mon[25205]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:16.085 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:15 smithi043 ceph-mon[29127]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:15 smithi043 ceph-mon[24430]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:18.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:17 smithi043 ceph-mon[29127]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:18.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:17 smithi043 ceph-mon[24430]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:18.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:17 smithi053 ceph-mon[25205]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:19.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:18 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:19.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:18 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:19.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:18 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:19.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:43:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:43:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:43:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:19 smithi053 ceph-mon[25205]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:20.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:19 smithi043 ceph-mon[29127]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:20.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:19 smithi043 ceph-mon[24430]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:21 smithi053 ceph-mon[25205]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:21.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:21 smithi043 ceph-mon[29127]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:21 smithi043 ceph-mon[24430]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:22.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:22 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:22 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:22 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:23 smithi043 ceph-mon[24430]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:23 smithi043 ceph-mon[29127]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:23 smithi053 ceph-mon[25205]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:23.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:43:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:43:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:43:24.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:43:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:43:24] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-15T15:43:26.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:25 smithi043 ceph-mon[24430]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:26.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:25 smithi043 ceph-mon[29127]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:26.129 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:25 smithi053 ceph-mon[25205]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:27 smithi043 ceph-mon[24430]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:27 smithi043 ceph-mon[29127]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:28.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:27 smithi053 ceph-mon[25205]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:29.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:43:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:43:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"172.21.15.53:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", cluster_addr=\"172.21.15.43\", device_class=\"ssd\", hostname=\"smithi043\", instance=\"smithi053:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.43\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:43:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:29 smithi043 ceph-mon[24430]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:30.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:29 smithi043 ceph-mon[29127]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:29 smithi053 ceph-mon[25205]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:31.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:31 smithi053 ceph-mon[25205]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:31.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:31 smithi043 ceph-mon[24430]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:31.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:31 smithi043 ceph-mon[29127]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:32.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:32 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:32.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:32 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:32.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:32 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:33.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:33 smithi043 ceph-mon[24430]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:33 smithi043 ceph-mon[29127]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:33 smithi053 ceph-mon[25205]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:33.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:43:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:43:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi043\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"22d82864-12cf-11ef-bc98-c7b262605968\", domainname=\"(none)\", instance=\"smithi043\", job=\"node\", machine=\"x86_64\", nodename=\"smithi043\", release=\"5.14.0-446.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-05-15T15:43:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:34 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:34.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:34 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:34 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:34.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:43:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:43:34] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-15T15:43:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:35 smithi043 ceph-mon[24430]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:35 smithi043 ceph-mon[29127]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:35 smithi053 ceph-mon[25205]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:38.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:37 smithi043 ceph-mon[24430]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:38.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:37 smithi043 ceph-mon[29127]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:37 smithi053 ceph-mon[25205]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:39.932 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:39 smithi053 ceph-mon[25205]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:40.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:39 smithi043 ceph-mon[24430]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:40.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:39 smithi043 ceph-mon[29127]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:41 smithi053 ceph-mon[25205]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:41.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:41 smithi043 ceph-mon[24430]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:41 smithi043 ceph-mon[29127]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:42.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:42 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:42 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:43:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:42 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:42 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:43:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:42 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:42 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:43:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:43 smithi043 ceph-mon[24430]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:43 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:43:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:43 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:43:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:43 smithi043 ceph-mon[24430]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:43:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:43 smithi043 ceph-mon[29127]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:43 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:43:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:43 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:43:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:43 smithi043 ceph-mon[29127]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:43:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:43 smithi053 ceph-mon[25205]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:43 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:43:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:43 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:43:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:43 smithi053 ceph-mon[25205]: from='mgr.24806 ' entity='mgr.x' 2024-05-15T15:43:44.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:43:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:43:44] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-15T15:43:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:45 smithi043 ceph-mon[24430]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:46.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:45 smithi043 ceph-mon[29127]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:46.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:45 smithi053 ceph-mon[25205]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:48.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:47 smithi043 ceph-mon[24430]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:48.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:47 smithi043 ceph-mon[29127]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:48.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:47 smithi053 ceph-mon[25205]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:48 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:49.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:48 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:49.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:48 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:43:50.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:49 smithi043 ceph-mon[24430]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:50.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:49 smithi043 ceph-mon[29127]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:49 smithi053 ceph-mon[25205]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:51.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:51 smithi043 ceph-mon[24430]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:51 smithi043 ceph-mon[29127]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:51 smithi053 ceph-mon[25205]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:52 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:52.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:52 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:52 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:43:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:53 smithi043 ceph-mon[24430]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:53 smithi043 ceph-mon[29127]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:53 smithi053 ceph-mon[25205]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:54.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:43:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:43:54] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-05-15T15:43:56.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:55 smithi043 ceph-mon[24430]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:56.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:55 smithi043 ceph-mon[29127]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:56.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:55 smithi053 ceph-mon[25205]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:43:58.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:57 smithi043 ceph-mon[24430]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:58.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:57 smithi043 ceph-mon[29127]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:43:58.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:57 smithi053 ceph-mon[25205]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:00.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:43:59 smithi043 ceph-mon[24430]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:00.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:43:59 smithi043 ceph-mon[29127]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:00.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:43:59 smithi053 ceph-mon[25205]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:01.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:01 smithi043 ceph-mon[24430]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:01 smithi043 ceph-mon[29127]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:01 smithi053 ceph-mon[25205]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:02 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:02 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:02.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:02 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:03.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:03 smithi043 ceph-mon[24430]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:03 smithi043 ceph-mon[29127]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:03 smithi053 ceph-mon[25205]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:04 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:04 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:04 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:04.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:04 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:44:04] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.33.4" 2024-05-15T15:44:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:05 smithi043 ceph-mon[24430]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:05 smithi043 ceph-mon[29127]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:05.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:05 smithi053 ceph-mon[25205]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:07 smithi043 ceph-mon[24430]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:08.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:07 smithi043 ceph-mon[29127]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:07 smithi053 ceph-mon[25205]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:09.932 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:09 smithi053 ceph-mon[25205]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:10.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:09 smithi043 ceph-mon[24430]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:09 smithi043 ceph-mon[29127]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:11 smithi043 ceph-mon[24430]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:11 smithi043 ceph-mon[29127]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:11 smithi053 ceph-mon[25205]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:12.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:12 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:12.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:12 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:12.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:12 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:13 smithi043 ceph-mon[29127]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:13 smithi043 ceph-mon[24430]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:13 smithi053 ceph-mon[25205]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:14.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:44:14] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.33.4" 2024-05-15T15:44:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:15 smithi043 ceph-mon[24430]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:16.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:15 smithi043 ceph-mon[29127]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:16.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:15 smithi053 ceph-mon[25205]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:18.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:17 smithi043 ceph-mon[24430]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:18.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:17 smithi043 ceph-mon[29127]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:18.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:17 smithi053 ceph-mon[25205]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:19.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:18 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:19.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:18 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:19.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:18 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:20.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:19 smithi043 ceph-mon[24430]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:20.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:19 smithi043 ceph-mon[29127]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:19 smithi053 ceph-mon[25205]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:21 smithi043 ceph-mon[24430]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:21 smithi043 ceph-mon[29127]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:21 smithi053 ceph-mon[25205]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:22 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:23.263 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:22 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:23.264 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:22 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:23 smithi043 ceph-mon[24430]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:23 smithi043 ceph-mon[29127]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:23 smithi053 ceph-mon[25205]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:24.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:44:24] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.33.4" 2024-05-15T15:44:26.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:25 smithi043 ceph-mon[24430]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:26.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:25 smithi043 ceph-mon[29127]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:26.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:25 smithi053 ceph-mon[25205]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:28.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:27 smithi043 ceph-mon[24430]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:27 smithi043 ceph-mon[29127]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:28.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:27 smithi053 ceph-mon[25205]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:30.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:29 smithi043 ceph-mon[24430]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:30.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:29 smithi043 ceph-mon[29127]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:29 smithi053 ceph-mon[25205]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:30.461 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T15:44:31.666 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:31 smithi043 ceph-mon[24430]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:31.666 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:31 smithi043 ceph-mon[29127]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:31.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:31 smithi053 ceph-mon[25205]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:32.490 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:44:32.490 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (5m) 2m ago 13m 24.1M - 0.23.0 ba2b418f427c dd5a5b0f3fe5 2024-05-15T15:44:32.490 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (5m) 3m ago 13m 44.5M - 8.3.5 dad864ee21e9 3fece04315b1 2024-05-15T15:44:32.491 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 running (5m) 2m ago 12m 51.2M - 3.5 e1d6a67b021e daca6b662f1d 2024-05-15T15:44:32.491 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283 running (8m) 3m ago 18m 515M - 19.0.0-2576-gde5210d5 48bc18eb22d8 cfeda1bd207c 2024-05-15T15:44:32.491 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (2m) 2m ago 19m 67.0M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:44:32.491 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (19m) 2m ago 20m 59.1M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:44:32.491 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (18m) 3m ago 18m 51.7M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:44:32.491 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (18m) 2m ago 18m 49.5M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:44:32.491 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (6m) 2m ago 14m 15.0M - 1.3.1 1dbe0e931976 083a05fd3833 2024-05-15T15:44:32.492 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (5m) 3m ago 14m 17.2M - 1.3.1 1dbe0e931976 2c6976b8c648 2024-05-15T15:44:32.492 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (17m) 2m ago 17m 52.5M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:44:32.492 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (17m) 2m ago 17m 56.2M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:44:32.492 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (17m) 2m ago 17m 49.5M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:44:32.492 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (16m) 2m ago 16m 51.7M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:44:32.492 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (16m) 3m ago 16m 50.3M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:44:32.493 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (15m) 3m ago 15m 49.2M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:44:32.493 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (15m) 3m ago 15m 51.5M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:44:32.493 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (15m) 3m ago 15m 53.5M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:44:32.493 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (5m) 3m ago 14m 70.7M - 2.33.4 514e6a882f6e e8019f2a6456 2024-05-15T15:44:32.493 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (13m) 2m ago 12m 72.8M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:44:32.493 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (12m) 3m ago 12m 71.4M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:44:32.503 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:32 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:32.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:32 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:32.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:32 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:33.109 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions' 2024-05-15T15:44:33.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:33 smithi043 ceph-mon[24430]: from='client.25063 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:44:33.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:33 smithi043 ceph-mon[24430]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:33.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:33 smithi043 ceph-mon[29127]: from='client.25063 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:44:33.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:33 smithi043 ceph-mon[29127]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:33 smithi053 ceph-mon[25205]: from='client.25063 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:44:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:33 smithi053 ceph-mon[25205]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:34.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:34 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:34.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:34 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:34 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:34.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:44:34] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-05-15T15:44:35.432 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:44:35.432 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:44:35.432 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:44:35.432 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:44:35.432 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:44:35.432 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:44:35.433 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:44:35.750 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:35 smithi043 ceph-mon[24430]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:35.750 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:35 smithi043 ceph-mon[29127]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:35 smithi053 ceph-mon[25205]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:36.082 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T15:44:36.733 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:36 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2348454179' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:44:36.733 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:36 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2348454179' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:44:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:36 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2348454179' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:44:37.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:37 smithi043 ceph-mon[24430]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:37.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:37 smithi043 ceph-mon[29127]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:37 smithi053 ceph-mon[25205]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:38.090 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_OK 2024-05-15T15:44:38.686 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:38 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3224267350' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:44:38.686 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:38 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3224267350' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:44:38.712 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph -s' 2024-05-15T15:44:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:38 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3224267350' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:44:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:39 smithi043 ceph-mon[24430]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:39.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:39 smithi043 ceph-mon[29127]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:39.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:39 smithi053 ceph-mon[25205]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:40.733 INFO:teuthology.orchestra.run.smithi043.stdout: cluster: 2024-05-15T15:44:40.733 INFO:teuthology.orchestra.run.smithi043.stdout: id: 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:44:40.733 INFO:teuthology.orchestra.run.smithi043.stdout: health: HEALTH_OK 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: services: 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: mgr: x(active, since 6m), standbys: y 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: data: 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: pools: 6 pools, 161 pgs 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: objects: 209 objects, 585 KiB 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: pgs: 161 active+clean 2024-05-15T15:44:40.734 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:44:40.735 INFO:teuthology.orchestra.run.smithi043.stdout: io: 2024-05-15T15:44:40.735 INFO:teuthology.orchestra.run.smithi043.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-15T15:44:40.735 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:44:41.367 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph mgr fail' 2024-05-15T15:44:41.683 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:41 smithi043 ceph-mon[29127]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:41.683 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:41 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3605844785' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:44:41.684 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:41 smithi043 ceph-mon[24430]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:41.684 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:41 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3605844785' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:44:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:41 smithi053 ceph-mon[25205]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:44:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:41 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3605844785' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:44:42.607 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:42 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:42.607 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:42 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:42 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:43.653 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:43 smithi043 ceph-mon[24430]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:43.653 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:43 smithi043 ceph-mon[24430]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:44:43.653 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:43 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3521586575' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:44:43.653 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:43 smithi043 ceph-mon[24430]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:44:43.653 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:43 smithi043 ceph-mon[24430]: osdmap e104: 8 total, 8 up, 8 in 2024-05-15T15:44:43.654 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:43 smithi043 ceph-mon[29127]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:43.654 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:43 smithi043 ceph-mon[29127]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:44:43.654 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:43 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3521586575' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:44:43.654 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:43 smithi043 ceph-mon[29127]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:44:43.654 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:43 smithi043 ceph-mon[29127]: osdmap e104: 8 total, 8 up, 8 in 2024-05-15T15:44:43.654 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:44:43] ENGINE Bus STOPPING 2024-05-15T15:44:43.655 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:44:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-15T15:44:43.655 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:44:43] ENGINE Bus STOPPED 2024-05-15T15:44:43.655 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:44:43] ENGINE Bus STARTING 2024-05-15T15:44:43.655 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:44:43] ENGINE Serving on http://:::9283 2024-05-15T15:44:43.655 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:44:43] ENGINE Bus STARTED 2024-05-15T15:44:43.675 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:43 smithi053 ceph-mon[25205]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:44:43.675 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:43 smithi053 ceph-mon[25205]: from='mgr.24806 172.21.15.53:0/3394921544' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:44:43.675 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:43 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3521586575' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:44:43.675 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:43 smithi053 ceph-mon[25205]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:44:43.675 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:43 smithi053 ceph-mon[25205]: osdmap e104: 8 total, 8 up, 8 in 2024-05-15T15:44:43.675 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ignoring --setuser ceph since I am not root 2024-05-15T15:44:43.676 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ignoring --setgroup ceph since I am not root 2024-05-15T15:44:43.676 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:43 smithi053 ceph-mgr[45196]: -- 172.21.15.53:0/1016013661 <== mon.2 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559b2b2485a0 con 0x559b2b21c800 2024-05-15T15:44:43.676 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:43.591+0000 7fe2ac011200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:44:44.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:43.792+0000 7fe2ac011200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:44:44.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:43.909+0000 7fe2ac011200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:44:44.191 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'sleep 180' 2024-05-15T15:44:44.549 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-15T15:44:44.549 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: mgrmap e28: y(active, starting, since 0.0305109s) 2024-05-15T15:44:44.549 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:44:44.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:44:44.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:44:44.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:44:44.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: Manager daemon y is now available 2024-05-15T15:44:44.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:44:44.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:44.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:44:44.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:44:44.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:44:44.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:44:44.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: mgrmap e28: y(active, starting, since 0.0305109s) 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:44:44.552 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:44:44.553 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:44:44.553 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:44:44.553 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:44:44.553 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: Manager daemon y is now available 2024-05-15T15:44:44.553 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:44:44.553 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:44.553 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:44:44.553 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:44:44.553 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:44:44.553 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:44 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:44:44.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:44.242+0000 7fe2ac011200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:44:44.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:44.341+0000 7fe2ac011200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:44:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-15T15:44:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: mgrmap e28: y(active, starting, since 0.0305109s) 2024-05-15T15:44:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:44:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:44:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:44:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:44:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:44:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:44:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:44:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: Manager daemon y is now available 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:44:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:44:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:44 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:44:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:45 smithi053 ceph-mon[25205]: mgrmap e29: y(active, since 1.04263s) 2024-05-15T15:44:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:45 smithi053 ceph-mon[25205]: [15/May/2024:15:44:44] ENGINE Bus STARTING 2024-05-15T15:44:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:45 smithi053 ceph-mon[25205]: [15/May/2024:15:44:44] ENGINE Serving on http://172.21.15.43:8765 2024-05-15T15:44:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:45 smithi053 ceph-mon[25205]: [15/May/2024:15:44:44] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:44:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:45 smithi053 ceph-mon[25205]: [15/May/2024:15:44:44] ENGINE Bus STARTED 2024-05-15T15:44:45.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:45 smithi043 ceph-mon[24430]: mgrmap e29: y(active, since 1.04263s) 2024-05-15T15:44:45.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:45 smithi043 ceph-mon[24430]: [15/May/2024:15:44:44] ENGINE Bus STARTING 2024-05-15T15:44:45.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:45 smithi043 ceph-mon[24430]: [15/May/2024:15:44:44] ENGINE Serving on http://172.21.15.43:8765 2024-05-15T15:44:45.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:45 smithi043 ceph-mon[24430]: [15/May/2024:15:44:44] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:44:45.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:45 smithi043 ceph-mon[24430]: [15/May/2024:15:44:44] ENGINE Bus STARTED 2024-05-15T15:44:45.675 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:45 smithi043 ceph-mon[29127]: mgrmap e29: y(active, since 1.04263s) 2024-05-15T15:44:45.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:45 smithi043 ceph-mon[29127]: [15/May/2024:15:44:44] ENGINE Bus STARTING 2024-05-15T15:44:45.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:45 smithi043 ceph-mon[29127]: [15/May/2024:15:44:44] ENGINE Serving on http://172.21.15.43:8765 2024-05-15T15:44:45.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:45 smithi043 ceph-mon[29127]: [15/May/2024:15:44:44] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:44:45.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:45 smithi043 ceph-mon[29127]: [15/May/2024:15:44:44] ENGINE Bus STARTED 2024-05-15T15:44:46.049 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:45.646+0000 7fe2ac011200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:44:46.049 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:45.841+0000 7fe2ac011200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:44:46.050 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:46.047+0000 7fe2ac011200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:44:46.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:46 smithi043 ceph-mon[24430]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:44:46.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:46 smithi043 ceph-mon[29127]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:44:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:46 smithi053 ceph-mon[25205]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:44:46.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:46.361+0000 7fe2ac011200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:44:46.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:46.462+0000 7fe2ac011200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:44:47.009 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:46.676+0000 7fe2ac011200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:44:47.009 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:46.895+0000 7fe2ac011200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:44:47.010 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:46.798Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.53:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.53:8765: connect: connection refused" 2024-05-15T15:44:47.010 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:46.798Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.53:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.53:8765: connect: connection refused" 2024-05-15T15:44:47.010 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:46.799Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.53:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.53:8765: connect: connection refused" 2024-05-15T15:44:47.010 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:46.800Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.53:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.53:8765: connect: connection refused" 2024-05-15T15:44:47.010 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:46.800Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.53:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.53:8765: connect: connection refused" 2024-05-15T15:44:47.519 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:47 smithi043 ceph-mon[24430]: mgrmap e30: y(active, since 3s) 2024-05-15T15:44:47.519 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:47 smithi043 ceph-mon[29127]: mgrmap e30: y(active, since 3s) 2024-05-15T15:44:47.797 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:47 smithi053 ceph-mon[25205]: mgrmap e30: y(active, since 3s) 2024-05-15T15:44:47.797 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:47.555+0000 7fe2ac011200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:44:48.121 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:48.016+0000 7fe2ac011200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:44:48.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:48.119+0000 7fe2ac011200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:44:48.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:48.341+0000 7fe2ac011200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-15T15:44:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:48 smithi043 ceph-mon[29127]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:44:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:48 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:48 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:48.689 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:48 smithi043 ceph-mon[24430]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:44:48.689 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:48 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:48.689 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:48 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:48 smithi053 ceph-mon[25205]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:44:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:48 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:48 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:49.282 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:48.909+0000 7fe2ac011200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:44:49.282 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:49.013+0000 7fe2ac011200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:44:49.539 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:49.280+0000 7fe2ac011200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:44:49.539 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:49.428+0000 7fe2ac011200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:44:49.539 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:49.537+0000 7fe2ac011200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:44:49.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:49.661+0000 7fe2ac011200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:44:50.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:49.980+0000 7fe2ac011200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:44:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:50 smithi043 ceph-mon[24430]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:44:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:50 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:50 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:50 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:50 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:50 smithi043 ceph-mon[29127]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:44:50.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:50 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:50 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:50 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:50 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.854 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:50.620+0000 7fe2ac011200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:44:50.855 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:50 smithi053 ceph-mon[25205]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:44:50.855 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:50 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.855 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:50 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.855 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:50 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:50.855 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:50 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:51.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:50.852+0000 7fe2ac011200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:44:51.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:44:50] ENGINE Bus STARTING 2024-05-15T15:44:51.131 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: CherryPy Checker: 2024-05-15T15:44:51.131 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: The Application mounted at '' has an empty config. 2024-05-15T15:44:51.131 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:44:51.131 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:44:50] ENGINE Serving on http://:::9283 2024-05-15T15:44:51.131 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:44:50] ENGINE Bus STARTED 2024-05-15T15:44:51.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[29127]: Standby manager daemon x started 2024-05-15T15:44:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:44:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:44:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:44:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:44:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:44:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:44:51.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[24430]: Standby manager daemon x started 2024-05-15T15:44:51.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:44:51.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:44:51.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:44:51.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:44:51.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:51.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:51.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:44:51.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:51 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:44:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:51 smithi053 ceph-mon[25205]: Standby manager daemon x started 2024-05-15T15:44:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:51 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:44:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:51 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:44:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:51 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:44:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:51 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:44:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:51 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:51 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:51 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:44:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:51 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:44:52.756 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: mgrmap e31: y(active, since 8s), standbys: x 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:44:52.757 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.758 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.758 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.758 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.758 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.758 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:44:52.758 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:44:52.758 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:44:52.758 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:44:52.758 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:44:52.758 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: mgrmap e31: y(active, since 8s), standbys: x 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.759 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.760 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:44:52.760 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:44:52.760 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:52 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:44:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:44:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:44:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:44:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:44:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: mgrmap e31: y(active, since 8s), standbys: x 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:44:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:44:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:52 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:44:53.760 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:53 smithi043 ceph-mon[24430]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:53.760 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:53 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:44:53.760 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:53 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:44:53.760 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:53 smithi043 ceph-mon[29127]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:53.760 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:53 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:44:53.760 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:53 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:44:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:53 smithi053 ceph-mon[25205]: from='client.24884 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:44:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:53 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:44:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:53 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:44:54.756 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:54 smithi043 ceph-mon[24430]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-15T15:44:54.756 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:54 smithi043 ceph-mon[29127]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-15T15:44:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:54 smithi053 ceph-mon[25205]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-15T15:44:54.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:44:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:44:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:44:55.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:44:55.534+0000 7f6971de7700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:44:55.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:44:55.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:44:55.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:44:55.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:44:55.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:44:55.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:55.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:44:55.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:44:55.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:55.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:44:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.651 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:56 smithi053 ceph-mon[25205]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:44:56.651 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:56 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:44:56.651 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:44:56.651 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:44:56.651 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:44:56.651 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:44:56.651 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.651 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.652 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:44:56.653 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.654 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.655 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:56 smithi053 ceph-mon[25205]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:44:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:56 smithi043 ceph-mon[24430]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:44:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:56 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:44:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:44:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:44:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:44:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:44:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:44:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:44:56.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:44:56.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:44:56.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:44:56.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:44:56.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:44:56.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:44:56.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:44:56.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:44:56.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:44:56.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:56 smithi043 ceph-mon[29127]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:56 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:44:56.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:44:56.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:44:56.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:44:56.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:44:56.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:44:56.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:44:56.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.845 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:56 smithi043 ceph-mon[29127]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:44:56.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:44:56.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:44:56.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:44:56.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:44:56.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:44:56.846 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:56 smithi043 ceph-mon[24430]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:44:57.654 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:57 smithi053 ceph-mon[25205]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:44:57.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:57 smithi043 ceph-mon[24430]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:44:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:57 smithi043 ceph-mon[29127]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:44:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:58 smithi053 ceph-mon[25205]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:44:58.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:58 smithi043 ceph-mon[24430]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:44:58.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:58 smithi043 ceph-mon[29127]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:44:59.380 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 systemd[1]: Stopping Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:44:59.380 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 bash[53714]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus.a" found: no such container 2024-05-15T15:44:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:44:59 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:44:59 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:44:59 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.619Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.619Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.619Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.619Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.619Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.619Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.619Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.619Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.619Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.620Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-15T15:44:59.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.620Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-15T15:44:59.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[48335]: ts=2024-05-15T15:44:59.620Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-15T15:44:59.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 podman[53726]: 2024-05-15 15:44:59.62969734 +0000 UTC m=+0.310649015 container died e8019f2a645644695438de85f667ba88a59ad2d43c2569a276f7fe95992ec296 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:45:00.153 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 podman[53726]: 2024-05-15 15:44:59.976258081 +0000 UTC m=+0.657209759 container cleanup e8019f2a645644695438de85f667ba88a59ad2d43c2569a276f7fe95992ec296 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:45:00.153 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:44:59 smithi053 bash[53726]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a 2024-05-15T15:45:00.411 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 podman[53737]: 2024-05-15 15:45:00.152270152 +0000 UTC m=+0.519327022 container remove e8019f2a645644695438de85f667ba88a59ad2d43c2569a276f7fe95992ec296 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:45:00.411 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 bash[53755]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus.a" found: no such container 2024-05-15T15:45:00.411 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-15T15:45:00.411 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 systemd[1]: Stopped Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:45:00.411 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Consumed 2.045s CPU time. 2024-05-15T15:45:00.411 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 systemd[1]: Starting Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:45:00.664 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:00 smithi053 ceph-mon[25205]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:45:00.664 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 podman[53817]: 2024-05-15 15:45:00.428280584 +0000 UTC m=+0.107681226 container create 4abe92292f7f3de121ed894126e164b5d903e4169ae90e94b75fabd14e6702d0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:45:00.665 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 podman[53817]: 2024-05-15 15:45:00.340299473 +0000 UTC m=+0.019700116 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-15T15:45:00.665 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 podman[53817]: 2024-05-15 15:45:00.663572887 +0000 UTC m=+0.342973529 container init 4abe92292f7f3de121ed894126e164b5d903e4169ae90e94b75fabd14e6702d0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:45:00.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:00 smithi043 ceph-mon[24430]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:45:00.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:00 smithi043 ceph-mon[29127]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:45:01.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 podman[53817]: 2024-05-15 15:45:00.667084187 +0000 UTC m=+0.346484829 container start 4abe92292f7f3de121ed894126e164b5d903e4169ae90e94b75fabd14e6702d0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:45:01.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.691Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-15T15:45:01.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.691Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.692Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.692Z caller=main.go:518 level=info host_details="(Linux 5.14.0-446.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024 x86_64 smithi053 (none))" 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.692Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.692Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.693Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.693Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.694Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.696Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.696Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.624µs 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.696Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 bash[53817]: 4abe92292f7f3de121ed894126e164b5d903e4169ae90e94b75fabd14e6702d0 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 systemd[1]: Started Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:45:01.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:00.896Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-05-15T15:45:01.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:01.147Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-05-15T15:45:01.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:01.356Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-05-15T15:45:01.630 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:01.356Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-05-15T15:45:01.631 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:01.356Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.079µs wal_replay_duration=659.720319ms total_replay_duration=659.769046ms 2024-05-15T15:45:01.631 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:01.361Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-15T15:45:01.631 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:01.361Z caller=main.go:947 level=info msg="TSDB started" 2024-05-15T15:45:01.631 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:01.361Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-15T15:45:01.631 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:01.390Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=29.16677ms db_storage=1.061µs remote_storage=1.362µs web_handler=312ns query_engine=697ns scrape=453.14µs scrape_sd=86.803µs notify=12.642µs notify_sd=21.113µs rules=28.276053ms 2024-05-15T15:45:01.631 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:45:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:45:01.390Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-15T15:45:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:45:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:45:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:45:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[24430]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:45:02.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:02.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:02.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:45:02.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:45:02.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:45:02.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:01 smithi043 ceph-mon[29127]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:45:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:01 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:01 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:01 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:45:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:01 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:45:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:01 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:45:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:01 smithi053 ceph-mon[25205]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-05-15T15:45:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:04 smithi043 ceph-mon[24430]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-15T15:45:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:04 smithi043 ceph-mon[29127]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-15T15:45:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:04 smithi053 ceph-mon[25205]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-15T15:45:06.142 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:06.142 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:06.142 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:45:06.142 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:45:06.142 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:06.142 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:45:06.142 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:45:06.142 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:45:06.143 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:45:06.143 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:45:06.143 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[24430]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-15T15:45:06.143 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:06.143 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:06.143 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:45:06.143 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:45:06.143 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:06.144 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:45:06.144 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:45:06.144 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:45:06.144 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:45:06.144 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:45:06.144 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:05 smithi043 ceph-mon[29127]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-15T15:45:06.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:06.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:06.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:45:06.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:45:06.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:45:06.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:45:06.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:45:06.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:45:06.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:45:06.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:45:06.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:05 smithi053 ceph-mon[25205]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-05-15T15:45:08.282 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:45:08.185+0000 7f6971de7700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:08.282 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:08.282 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:08.282 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:08.282 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:45:08.282 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:45:08.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:45:08.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:45:08.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:45:08.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:45:08.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:45:08.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:45:08.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:45:08.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:45:08.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:45:08.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:45:08.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:08.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:08.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:08 smithi043 ceph-mon[24430]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:45:08.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:08 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/220046916' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T15:45:08.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:08 smithi043 ceph-mon[29127]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:45:08.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:08 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/220046916' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T15:45:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:08 smithi053 ceph-mon[25205]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:45:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:08 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/220046916' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T15:45:09.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:09 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:45:09.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:45:09.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:45:09.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:45:09.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:45:09.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:45:09.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:45:09.591 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:45:09.591 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:45:09.591 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:45:09.591 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:45:09.591 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:09 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:45:09.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:45:09.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:45:09.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:45:09.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:45:09.597 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:45:09.597 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:45:09.597 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.597 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:09 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.781 INFO:journalctl@ceph.mon.b.smithi053.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:45:09.782 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:45:09.783 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.784 INFO:journalctl@ceph.mon.b.smithi053.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi-foo-smithi043-aclusg 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: /usr/bin/podman: stderr Error: no such container ceph-22d82864-12cf-11ef-bc98-c7b262605968-iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:45:09.785 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:45:09.786 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:45:09.786 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:45:09.786 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:45:09.786 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:45:09.786 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:45:09.786 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:45:09.786 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:45:09.786 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:45:09.786 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:45:09.786 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:45:10.793 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:10 smithi053 ceph-mon[25205]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:45:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:10 smithi043 ceph-mon[24430]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:45:10.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:10 smithi043 ceph-mon[29127]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:45:12.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:12 smithi043 ceph-mon[24430]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-15T15:45:12.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:12 smithi043 ceph-mon[29127]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-15T15:45:12.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:12 smithi053 ceph-mon[25205]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-15T15:45:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:14 smithi043 ceph-mon[24430]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-15T15:45:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:14 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:14 smithi043 ceph-mon[29127]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-15T15:45:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:14 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:14 smithi053 ceph-mon[25205]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-15T15:45:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:14 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:16.638 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:16 smithi043 ceph-mon[24430]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-15T15:45:16.639 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:16 smithi043 ceph-mon[29127]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-15T15:45:16.639 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:45:16] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-15T15:45:16.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:16 smithi053 ceph-mon[25205]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-15T15:45:18.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:18 smithi043 ceph-mon[24430]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-15T15:45:18.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:18 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:18.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:18 smithi043 ceph-mon[29127]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-15T15:45:18.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:18 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:18 smithi053 ceph-mon[25205]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-15T15:45:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:18 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:20.835 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:20 smithi053 ceph-mon[25205]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-15T15:45:20.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:20 smithi043 ceph-mon[24430]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-15T15:45:20.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:20 smithi043 ceph-mon[29127]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-05-15T15:45:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:22 smithi043 ceph-mon[24430]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-15T15:45:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:22 smithi043 ceph-mon[29127]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-15T15:45:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:22 smithi053 ceph-mon[25205]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-05-15T15:45:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:24 smithi043 ceph-mon[24430]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:24.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:24 smithi043 ceph-mon[29127]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:24.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:24 smithi053 ceph-mon[25205]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:26 smithi043 ceph-mon[24430]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:26 smithi043 ceph-mon[29127]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:26.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:45:26] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-15T15:45:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:26 smithi053 ceph-mon[25205]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:28.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:28 smithi043 ceph-mon[24430]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:28.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:28 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:28.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:28 smithi043 ceph-mon[29127]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:28.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:28 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:28 smithi053 ceph-mon[25205]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:28 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:29 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:29 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:29 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:30 smithi043 ceph-mon[24430]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:30.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:30 smithi043 ceph-mon[29127]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:30.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:30 smithi053 ceph-mon[25205]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:32.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:32 smithi043 ceph-mon[24430]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:32.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:32 smithi043 ceph-mon[29127]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:32.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:32 smithi053 ceph-mon[25205]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:34 smithi043 ceph-mon[24430]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:34.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:34 smithi043 ceph-mon[29127]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:34 smithi053 ceph-mon[25205]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:36.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:36 smithi043 ceph-mon[24430]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:36.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:36 smithi043 ceph-mon[29127]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:36.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:45:36] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-05-15T15:45:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:36 smithi053 ceph-mon[25205]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:38.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:38 smithi043 ceph-mon[24430]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:38.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:38 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:38 smithi043 ceph-mon[29127]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:38 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:38 smithi053 ceph-mon[25205]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:38 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:40 smithi043 ceph-mon[24430]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:40 smithi043 ceph-mon[29127]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:40 smithi053 ceph-mon[25205]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:42 smithi043 ceph-mon[24430]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:42 smithi043 ceph-mon[29127]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:42 smithi053 ceph-mon[25205]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:44 smithi043 ceph-mon[24430]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:44 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:44 smithi043 ceph-mon[29127]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:44 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:44 smithi053 ceph-mon[25205]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:44 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:46.638 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:46 smithi043 ceph-mon[24430]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:46.638 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:46 smithi043 ceph-mon[29127]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:46.639 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:45:46] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-05-15T15:45:46.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:46 smithi053 ceph-mon[25205]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:48 smithi043 ceph-mon[24430]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:48 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:48.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:48 smithi043 ceph-mon[29127]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:48 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:48 smithi053 ceph-mon[25205]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:48 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:50 smithi043 ceph-mon[24430]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:50.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:50 smithi043 ceph-mon[29127]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:50 smithi053 ceph-mon[25205]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:52.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:52 smithi043 ceph-mon[24430]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:52.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:52 smithi043 ceph-mon[29127]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:52 smithi053 ceph-mon[25205]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:54.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:54 smithi043 ceph-mon[24430]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:54 smithi043 ceph-mon[29127]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:54 smithi053 ceph-mon[25205]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:56 smithi043 ceph-mon[24430]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:56.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:56 smithi043 ceph-mon[29127]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:56.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:45:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:45:56] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-05-15T15:45:56.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:56 smithi053 ceph-mon[25205]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:45:58.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:58 smithi043 ceph-mon[24430]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:58.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:58 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:58.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:58 smithi043 ceph-mon[29127]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:58.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:58 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:58.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:58 smithi053 ceph-mon[25205]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:45:58.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:58 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:45:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:45:59 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:45:59 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:45:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:45:59 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:00 smithi043 ceph-mon[24430]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:00.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:00 smithi043 ceph-mon[29127]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:00 smithi053 ceph-mon[25205]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:02 smithi043 ceph-mon[24430]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:02 smithi043 ceph-mon[29127]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:02.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:02 smithi053 ceph-mon[25205]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:04 smithi043 ceph-mon[24430]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:04 smithi043 ceph-mon[29127]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:04 smithi053 ceph-mon[25205]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:06.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:06 smithi043 ceph-mon[24430]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:06.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:46:06] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-05-15T15:46:06.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:06 smithi043 ceph-mon[29127]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:06.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:06 smithi053 ceph-mon[25205]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:08 smithi043 ceph-mon[29127]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:08 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:08 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:46:08.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:08 smithi043 ceph-mon[24430]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:08.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:08 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:08.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:08 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:46:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:08 smithi053 ceph-mon[25205]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:08 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:08 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:46:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:46:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:46:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:46:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:46:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:09 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:46:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:09 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:09 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:09 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:09 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:09 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:46:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:09 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:46:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:09 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:46:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:09 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:46:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:09 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:46:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:09 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:46:11.031 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:10 smithi053 ceph-mon[25205]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:11.032 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:10 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:46:11.032 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:10 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:46:11.067 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:10 smithi043 ceph-mon[24430]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:11.067 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:10 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:46:11.067 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:10 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:46:11.067 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:10 smithi043 ceph-mon[29127]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:11.068 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:10 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:46:11.068 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:10 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:46:11.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:46:11.204+0000 7f6971de7700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:46:11.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:46:11.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:46:11.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:46:11.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:46:11.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:46:11.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:46:11.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:11.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:11.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:46:11.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:46:11.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:46:11.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:46:11.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:46:11.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:46:11.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:46:11.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:46:11.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:46:11.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:11.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:12 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:46:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:46:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:46:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:46:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:46:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:46:13.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:46:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:46:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:12 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:46:13.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:46:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:46:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:46:13.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:46:13.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:46:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:46:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:46:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:46:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:12 smithi043 ceph-mon[24430]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:13.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:46:13.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:46:13.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:46:13.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:12 smithi043 ceph-mon[29127]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:13.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:12 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:46:13.130 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:46:13.130 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:46:13.130 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:46:13.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:46:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:46:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:46:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:46:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:12 smithi053 ceph-mon[25205]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:14.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:13 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:14.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:13 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:14.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:13 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:15.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:14 smithi043 ceph-mon[24430]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:15.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:14 smithi043 ceph-mon[29127]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:14 smithi053 ceph-mon[25205]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:16.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:16 smithi043 ceph-mon[24430]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:16.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:16 smithi043 ceph-mon[29127]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:16.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:46:16] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-15T15:46:16.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:16 smithi053 ceph-mon[25205]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:18.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:18 smithi043 ceph-mon[24430]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:18.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:18 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:18.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:18 smithi043 ceph-mon[29127]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:18.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:18 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:18 smithi053 ceph-mon[25205]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:18 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:20.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:20 smithi043 ceph-mon[24430]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:20.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:20 smithi043 ceph-mon[29127]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:20 smithi053 ceph-mon[25205]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:22 smithi043 ceph-mon[24430]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:22 smithi043 ceph-mon[29127]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:22.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:22 smithi053 ceph-mon[25205]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:24 smithi043 ceph-mon[24430]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:24.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:24 smithi043 ceph-mon[29127]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:24.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:24 smithi053 ceph-mon[25205]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:26 smithi043 ceph-mon[24430]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:26 smithi043 ceph-mon[29127]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:26.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:46:26] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-05-15T15:46:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:26 smithi053 ceph-mon[25205]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:28.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:28 smithi043 ceph-mon[24430]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:28.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:28 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:28.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:28 smithi043 ceph-mon[29127]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:28.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:28 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:28 smithi053 ceph-mon[25205]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:28 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:29 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:29 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:29 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:30 smithi043 ceph-mon[24430]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:30.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:30 smithi043 ceph-mon[29127]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:30.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:30 smithi053 ceph-mon[25205]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:32.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:32 smithi043 ceph-mon[24430]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:32.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:32 smithi043 ceph-mon[29127]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:32.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:32 smithi053 ceph-mon[25205]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:34 smithi043 ceph-mon[24430]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:34.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:34 smithi043 ceph-mon[29127]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:34 smithi053 ceph-mon[25205]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:36.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:36 smithi043 ceph-mon[24430]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:36.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:36 smithi043 ceph-mon[29127]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:36.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:46:36] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-05-15T15:46:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:36 smithi053 ceph-mon[25205]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:38.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:38 smithi043 ceph-mon[24430]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:38.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:38 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:38 smithi043 ceph-mon[29127]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:38 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:38 smithi053 ceph-mon[25205]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:38 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:40 smithi043 ceph-mon[24430]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:40 smithi043 ceph-mon[29127]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:40 smithi053 ceph-mon[25205]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:42 smithi043 ceph-mon[24430]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:42 smithi043 ceph-mon[29127]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:42.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:42 smithi053 ceph-mon[25205]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:43 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:43 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:43 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:44 smithi043 ceph-mon[24430]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:44 smithi043 ceph-mon[29127]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:44 smithi053 ceph-mon[25205]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:46.638 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:46 smithi043 ceph-mon[24430]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:46.639 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:46 smithi043 ceph-mon[29127]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:46.639 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:46:46] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-05-15T15:46:46.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:46 smithi053 ceph-mon[25205]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:48 smithi043 ceph-mon[24430]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:48 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:48.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:48 smithi043 ceph-mon[29127]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:48.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:48 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:48 smithi053 ceph-mon[25205]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:48 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:50 smithi043 ceph-mon[24430]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:50.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:50 smithi043 ceph-mon[29127]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:50 smithi053 ceph-mon[25205]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:52 smithi043 ceph-mon[24430]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:52.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:52 smithi043 ceph-mon[29127]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:52 smithi053 ceph-mon[25205]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:54.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:54 smithi043 ceph-mon[24430]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:54 smithi043 ceph-mon[29127]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:54 smithi053 ceph-mon[25205]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:56 smithi043 ceph-mon[24430]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:56.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:56 smithi043 ceph-mon[29127]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:56.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:46:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:46:56] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-05-15T15:46:56.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:56 smithi053 ceph-mon[25205]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:46:58.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:58 smithi043 ceph-mon[24430]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:58.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:58 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:58.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:46:58 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:58.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:58 smithi043 ceph-mon[29127]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:58.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:58 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:58.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:46:58 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:46:58.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:58 smithi053 ceph-mon[25205]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:46:58.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:58 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:46:58.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:46:58 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:00 smithi043 ceph-mon[24430]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:00.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:00 smithi043 ceph-mon[29127]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:00 smithi053 ceph-mon[25205]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:02 smithi043 ceph-mon[24430]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:02 smithi043 ceph-mon[29127]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:02.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:02 smithi053 ceph-mon[25205]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:04 smithi043 ceph-mon[24430]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:04 smithi043 ceph-mon[29127]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:04 smithi053 ceph-mon[25205]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:06.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:06 smithi043 ceph-mon[24430]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:06.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:06 smithi043 ceph-mon[29127]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:06.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:47:06] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-05-15T15:47:06.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:06 smithi053 ceph-mon[25205]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:08 smithi043 ceph-mon[24430]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:08 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:08 smithi043 ceph-mon[29127]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:08 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:08 smithi053 ceph-mon[25205]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:08 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:10 smithi043 ceph-mon[24430]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:10.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:10 smithi043 ceph-mon[29127]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:10.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:10 smithi053 ceph-mon[25205]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:11.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:11 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:47:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:11 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:47:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:11 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:47:12.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:12 smithi053 ceph-mon[25205]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:12.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:12 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:47:12.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:12 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:47:12.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:12 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:47:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:12 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:47:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:12 smithi053 ceph-mon[25205]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:47:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:12 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:47:12.816 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[24430]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:12.816 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:47:12.816 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:47:12.816 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:47:12.817 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:47:12.817 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[24430]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:47:12.817 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:47:12.817 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[29127]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:12.817 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:47:12.817 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:47:12.817 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' 2024-05-15T15:47:12.817 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:47:12.818 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[29127]: from='mgr.25057 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:47:12.818 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:12 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:47:13.776 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:13 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:47:13.776 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:13 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:47:13.776 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:13 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:13.777 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:13 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:47:13.777 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:13 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:47:13.777 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:13 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:13 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:47:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:13 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:47:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:13 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:14.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:47:13.885+0000 7f6971de7700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:47:14.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:47:14.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:47:14.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:47:14.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:47:14.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:47:14.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:47:14.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:14.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:47:14.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:14.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:47:14.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:14.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:14 smithi043 ceph-mon[24430]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:14 smithi043 ceph-mon[29127]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:14 smithi053 ceph-mon[25205]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:15 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:47:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:47:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:47:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:47:15.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:47:15.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:47:15.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:47:15.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:47:15.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:47:15.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:47:15.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:47:15.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:47:15.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:47:15.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:15 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:47:15.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:47:15.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:47:15.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:47:15.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:47:15.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:47:15.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:47:15.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:47:15.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:47:15.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:47:15.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:47:15.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:47:15.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:47:15.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:15 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:47:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:47:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:47:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:47:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:47:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:47:15.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:47:15.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:47:15.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:47:15.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:47:15.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:47:16.737 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:16 smithi043 ceph-mon[24430]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:16.738 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:16 smithi043 ceph-mon[29127]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:16.739 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:47:16] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-15T15:47:16.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:16 smithi053 ceph-mon[25205]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:18.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:18 smithi043 ceph-mon[24430]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:18.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:18 smithi043 ceph-mon[29127]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:18 smithi053 ceph-mon[25205]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:19 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:19 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:19 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:20.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:20 smithi043 ceph-mon[24430]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:20.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:20 smithi043 ceph-mon[29127]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:20 smithi053 ceph-mon[25205]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:22 smithi043 ceph-mon[24430]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:22 smithi043 ceph-mon[29127]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:22.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:22 smithi053 ceph-mon[25205]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:24 smithi043 ceph-mon[24430]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:24.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:24 smithi043 ceph-mon[29127]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:24.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:24 smithi053 ceph-mon[25205]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:26 smithi043 ceph-mon[24430]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:26 smithi043 ceph-mon[29127]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:26.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:47:26] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-05-15T15:47:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:26 smithi053 ceph-mon[25205]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:28.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:28 smithi043 ceph-mon[24430]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:28.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:28 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:28.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:28 smithi043 ceph-mon[29127]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:28.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:28 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:28 smithi053 ceph-mon[25205]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:28 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:29 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:29 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:29 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:30.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:30 smithi043 ceph-mon[29127]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:30 smithi043 ceph-mon[24430]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:30.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:30 smithi053 ceph-mon[25205]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:32.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:32 smithi043 ceph-mon[24430]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:32.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:32 smithi043 ceph-mon[29127]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:32.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:32 smithi053 ceph-mon[25205]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:34 smithi043 ceph-mon[24430]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:34.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:34 smithi043 ceph-mon[29127]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:34 smithi053 ceph-mon[25205]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:36.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:36 smithi043 ceph-mon[24430]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:36.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:36 smithi043 ceph-mon[29127]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:36.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:47:36] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-05-15T15:47:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:36 smithi053 ceph-mon[25205]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:38.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:38 smithi043 ceph-mon[24430]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:38 smithi043 ceph-mon[29127]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:38 smithi053 ceph-mon[25205]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:39 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:39 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:39 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:40 smithi043 ceph-mon[24430]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:40 smithi043 ceph-mon[29127]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:40 smithi053 ceph-mon[25205]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:42 smithi043 ceph-mon[24430]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:42 smithi043 ceph-mon[29127]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:42 smithi053 ceph-mon[25205]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:43 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:43 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:43 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:44 smithi043 ceph-mon[24430]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:44 smithi043 ceph-mon[29127]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:44 smithi053 ceph-mon[25205]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:46.547 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:46 smithi043 ceph-mon[29127]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:46.547 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:46 smithi043 ceph-mon[24430]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:46.548 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:47:46] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-05-15T15:47:46.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:46 smithi053 ceph-mon[25205]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:47.019 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T15:47:48.808 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:48 smithi043 ceph-mon[24430]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:48.808 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:48 smithi043 ceph-mon[29127]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:48 smithi053 ceph-mon[25205]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:49.157 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:47:49.157 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (9m) 2m ago 16m 24.2M - 0.23.0 ba2b418f427c dd5a5b0f3fe5 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (8m) 2m ago 16m 52.0M - 8.3.5 dad864ee21e9 3fece04315b1 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 running (9m) 2m ago 16m 59.1M - 3.5 e1d6a67b021e daca6b662f1d 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283 running (11m) 2m ago 21m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 cfeda1bd207c 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (6m) 2m ago 23m 478M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (23m) 2m ago 23m 60.3M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (21m) 2m ago 21m 51.6M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (21m) 2m ago 21m 43.8M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (9m) 2m ago 17m 15.7M - 1.3.1 1dbe0e931976 083a05fd3833 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (9m) 2m ago 17m 17.2M - 1.3.1 1dbe0e931976 2c6976b8c648 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (21m) 2m ago 21m 52.9M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (20m) 2m ago 20m 56.5M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (20m) 2m ago 20m 49.9M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:47:49.158 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (19m) 2m ago 19m 51.9M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:47:49.159 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (19m) 2m ago 19m 51.8M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:47:49.159 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (19m) 2m ago 19m 50.4M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:47:49.159 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (18m) 2m ago 18m 53.2M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:47:49.159 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (18m) 2m ago 18m 55.0M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:47:49.159 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (2m) 2m ago 17m 44.3M - 2.33.4 514e6a882f6e 4abe92292f7f 2024-05-15T15:47:49.159 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (16m) 2m ago 16m 73.9M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:47:49.159 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (16m) 2m ago 16m 73.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:47:49.662 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions' 2024-05-15T15:47:49.686 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:49 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:49.686 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:49 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:49 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:50 smithi043 ceph-mon[24430]: from='client.25144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:47:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:50 smithi043 ceph-mon[24430]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:50.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:50 smithi043 ceph-mon[29127]: from='client.25144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:47:50.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:50 smithi043 ceph-mon[29127]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:50 smithi053 ceph-mon[25205]: from='client.25144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:47:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:50 smithi053 ceph-mon[25205]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:51.870 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:47:51.870 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:47:51.870 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:47:51.870 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:47:51.870 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:47:51.871 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:47:52.440 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph -s' 2024-05-15T15:47:52.798 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:52 smithi043 ceph-mon[24430]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:52.799 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:52 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3483793858' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:47:52.799 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:52 smithi043 ceph-mon[29127]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:52.799 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:52 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3483793858' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:47:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:52 smithi053 ceph-mon[25205]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:52 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3483793858' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:47:54.807 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:54 smithi043 ceph-mon[24430]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:54.807 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:54 smithi043 ceph-mon[29127]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:54.807 INFO:teuthology.orchestra.run.smithi043.stdout: cluster: 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: id: 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: health: HEALTH_OK 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: services: 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: mgr: y(active, since 3m), standbys: x 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: data: 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: pools: 6 pools, 161 pgs 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: objects: 209 objects, 585 KiB 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: pgs: 161 active+clean 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:47:54.808 INFO:teuthology.orchestra.run.smithi043.stdout: io: 2024-05-15T15:47:54.809 INFO:teuthology.orchestra.run.smithi043.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-05-15T15:47:54.809 INFO:teuthology.orchestra.run.smithi043.stdout: 2024-05-15T15:47:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:54 smithi053 ceph-mon[25205]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:55.391 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T15:47:55.712 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:55 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2205696160' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:47:55.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:55 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2205696160' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:47:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:55 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2205696160' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-15T15:47:56.630 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:56 smithi043 ceph-mon[24430]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:56.631 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:56 smithi043 ceph-mon[29127]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:56.631 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:47:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:47:56] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-05-15T15:47:56.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:56 smithi053 ceph-mon[25205]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:47:57.401 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_OK 2024-05-15T15:47:57.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:57 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2560263191' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:47:57.704 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:57 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2560263191' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:47:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:57 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2560263191' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:47:58.038 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-05-15T15:47:58.682 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:58 smithi043 ceph-mon[24430]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:58.683 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:58 smithi043 ceph-mon[24430]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:58.683 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:58 smithi043 ceph-mon[29127]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:58.683 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:58 smithi043 ceph-mon[29127]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:58.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:58 smithi053 ceph-mon[25205]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:47:58.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:58 smithi053 ceph-mon[25205]: from='mgr.25057 172.21.15.43:0/1069694349' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:47:59.583 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:47:59 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:59.583 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:47:59 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:47:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:47:59 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:00.111 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:48:00.711 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:00 smithi043 ceph-mon[29127]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:00.711 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:00 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3445968329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:48:00.711 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:00 smithi043 ceph-mon[24430]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:00.711 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:00 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3445968329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:48:00.736 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph mgr fail' 2024-05-15T15:48:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:00 smithi053 ceph-mon[25205]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:00 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3445968329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:48:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:02 smithi043 ceph-mon[24430]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:02 smithi043 ceph-mon[29127]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:02.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:02 smithi053 ceph-mon[25205]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:03.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:03 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1244262005' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:48:03.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:03 smithi043 ceph-mon[24430]: osdmap e105: 8 total, 8 up, 8 in 2024-05-15T15:48:03.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:03.554+0000 7f69df830700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-15T15:48:03.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ignoring --setuser ceph since I am not root 2024-05-15T15:48:03.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ignoring --setgroup ceph since I am not root 2024-05-15T15:48:03.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:03 smithi043 ceph-mgr[62161]: -- 172.21.15.43:0/4068212210 <== mon.0 v2:172.21.15.43:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5585a3d405a0 con 0x5585a3d14800 2024-05-15T15:48:03.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:03.752+0000 7ffb38c60200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:48:03.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:03 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1244262005' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:48:03.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:03 smithi043 ceph-mon[29127]: osdmap e105: 8 total, 8 up, 8 in 2024-05-15T15:48:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:03 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1244262005' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-05-15T15:48:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:03 smithi053 ceph-mon[25205]: osdmap e105: 8 total, 8 up, 8 in 2024-05-15T15:48:03.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:48:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:48:03] ENGINE Bus STOPPING 2024-05-15T15:48:03.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:48:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:48:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-15T15:48:03.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:48:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:48:03] ENGINE Bus STOPPED 2024-05-15T15:48:03.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:48:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:48:03] ENGINE Bus STARTING 2024-05-15T15:48:03.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:48:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:48:03] ENGINE Serving on http://:::9283 2024-05-15T15:48:03.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:48:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:48:03] ENGINE Bus STARTED 2024-05-15T15:48:04.159 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:03.954+0000 7ffb38c60200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:48:04.160 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:04 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:04.069+0000 7ffb38c60200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:48:04.193 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'sleep 180' 2024-05-15T15:48:04.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1244262005' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-15T15:48:04.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: mgrmap e32: x(active, starting, since 0.799789s) 2024-05-15T15:48:04.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:48:04.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:48:04.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:48:04.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:48:04.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:48:04.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:48:04.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: Manager daemon x is now available 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:48:04.675 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:48:04.676 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:48:04.676 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:48:04.676 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:04 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:04.416+0000 7ffb38c60200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:48:04.676 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:04 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:04.517+0000 7ffb38c60200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:48:04.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1244262005' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-15T15:48:04.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: mgrmap e32: x(active, starting, since 0.799789s) 2024-05-15T15:48:04.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:48:04.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:48:04.676 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: Manager daemon x is now available 2024-05-15T15:48:04.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:48:04.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:04.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:48:04.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:48:04.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:48:04.678 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:04 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:48:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/1244262005' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-05-15T15:48:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: mgrmap e32: x(active, starting, since 0.799789s) 2024-05-15T15:48:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:48:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:48:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:48:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:48:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: Manager daemon x is now available 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:48:04.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:48:04.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:48:04.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:04 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-15T15:48:05.583 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:05 smithi043 ceph-mon[29127]: mgrmap e33: x(active, since 1.80603s) 2024-05-15T15:48:05.583 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:05 smithi043 ceph-mon[29127]: [15/May/2024:15:48:04] ENGINE Bus STARTING 2024-05-15T15:48:05.583 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:05 smithi043 ceph-mon[29127]: [15/May/2024:15:48:04] ENGINE Serving on http://172.21.15.53:8765 2024-05-15T15:48:05.583 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:05 smithi043 ceph-mon[29127]: [15/May/2024:15:48:04] ENGINE Serving on https://172.21.15.53:7150 2024-05-15T15:48:05.584 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:05 smithi043 ceph-mon[29127]: [15/May/2024:15:48:04] ENGINE Bus STARTED 2024-05-15T15:48:05.584 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:05 smithi043 ceph-mon[24430]: mgrmap e33: x(active, since 1.80603s) 2024-05-15T15:48:05.584 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:05 smithi043 ceph-mon[24430]: [15/May/2024:15:48:04] ENGINE Bus STARTING 2024-05-15T15:48:05.584 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:05 smithi043 ceph-mon[24430]: [15/May/2024:15:48:04] ENGINE Serving on http://172.21.15.53:8765 2024-05-15T15:48:05.584 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:05 smithi043 ceph-mon[24430]: [15/May/2024:15:48:04] ENGINE Serving on https://172.21.15.53:7150 2024-05-15T15:48:05.584 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:05 smithi043 ceph-mon[24430]: [15/May/2024:15:48:04] ENGINE Bus STARTED 2024-05-15T15:48:05.722 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:05 smithi053 ceph-mon[25205]: mgrmap e33: x(active, since 1.80603s) 2024-05-15T15:48:05.722 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:05 smithi053 ceph-mon[25205]: [15/May/2024:15:48:04] ENGINE Bus STARTING 2024-05-15T15:48:05.722 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:05 smithi053 ceph-mon[25205]: [15/May/2024:15:48:04] ENGINE Serving on http://172.21.15.53:8765 2024-05-15T15:48:05.722 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:05 smithi053 ceph-mon[25205]: [15/May/2024:15:48:04] ENGINE Serving on https://172.21.15.53:7150 2024-05-15T15:48:05.722 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:05 smithi053 ceph-mon[25205]: [15/May/2024:15:48:04] ENGINE Bus STARTED 2024-05-15T15:48:05.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:05.828+0000 7ffb38c60200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:48:06.201 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:06.014+0000 7ffb38c60200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:48:06.201 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:06.198+0000 7ffb38c60200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:48:06.516 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:06.514+0000 7ffb38c60200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:48:06.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:06 smithi043 ceph-mon[29127]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:48:06.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:06 smithi043 ceph-mon[29127]: mgrmap e34: x(active, since 2s) 2024-05-15T15:48:06.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:06 smithi043 ceph-mon[24430]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:48:06.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:06 smithi043 ceph-mon[24430]: mgrmap e34: x(active, since 2s) 2024-05-15T15:48:06.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:06.618+0000 7ffb38c60200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:48:06.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:06 smithi053 ceph-mon[25205]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:48:06.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:06 smithi053 ceph-mon[25205]: mgrmap e34: x(active, since 2s) 2024-05-15T15:48:07.109 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:06.834+0000 7ffb38c60200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:48:07.109 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:07 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:07.049+0000 7ffb38c60200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:48:08.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:07 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:07.717+0000 7ffb38c60200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:48:08.529 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:08.201+0000 7ffb38c60200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:48:08.529 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:08.306+0000 7ffb38c60200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:48:08.529 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:08.527+0000 7ffb38c60200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-15T15:48:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:08 smithi043 ceph-mon[29127]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:48:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:08 smithi043 ceph-mon[29127]: mgrmap e35: x(active, since 4s) 2024-05-15T15:48:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:08 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:08 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:08 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:08 smithi043 ceph-mon[24430]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:48:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:08 smithi043 ceph-mon[24430]: mgrmap e35: x(active, since 4s) 2024-05-15T15:48:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:08 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:08 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:08 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:08 smithi053 ceph-mon[25205]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:48:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:08 smithi053 ceph-mon[25205]: mgrmap e35: x(active, since 4s) 2024-05-15T15:48:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:08 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:08 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:08 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:09.200 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:09.096+0000 7ffb38c60200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:48:09.200 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:09.198+0000 7ffb38c60200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:48:09.729 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:09.467+0000 7ffb38c60200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:48:09.729 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:09.616+0000 7ffb38c60200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:48:09.729 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:09.727+0000 7ffb38c60200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:48:10.085 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:09 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:09.853+0000 7ffb38c60200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:48:10.511 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:10.176+0000 7ffb38c60200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:48:10.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:10 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:10 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:10 smithi043 ceph-mon[29127]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:48:10.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:10 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:10 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:10 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:10 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:10 smithi043 ceph-mon[24430]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:48:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:10 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:10 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:10 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:10 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:10 smithi053 ceph-mon[25205]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:48:10.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:10 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:10.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:10 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:11.193 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:10.842+0000 7ffb38c60200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:48:11.193 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:11.083+0000 7ffb38c60200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:48:11.193 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:48:11] ENGINE Bus STARTING 2024-05-15T15:48:11.193 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CherryPy Checker: 2024-05-15T15:48:11.193 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: The Application mounted at '' has an empty config. 2024-05-15T15:48:11.193 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:48:11.193 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:48:11] ENGINE Serving on http://:::9283 2024-05-15T15:48:11.193 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:48:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:48:11] ENGINE Bus STARTED 2024-05-15T15:48:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: Standby manager daemon y started 2024-05-15T15:48:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-15T15:48:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:48:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-15T15:48:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:48:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:48:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[24430]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: Standby manager daemon y started 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:11.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:11.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:48:11.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:48:11.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:48:11.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:48:11.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:48:11.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:11 smithi043 ceph-mon[29127]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: Standby manager daemon y started 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:48:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:11 smithi053 ceph-mon[25205]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:48:12.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: mgrmap e36: x(active, since 8s), standbys: y 2024-05-15T15:48:12.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:48:12.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:48:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:48:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:48:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:48:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:48:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.783 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 15 op/s 2024-05-15T15:48:12.783 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.783 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:12 smithi053 ceph-mon[25205]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:48:12.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: mgrmap e36: x(active, since 8s), standbys: y 2024-05-15T15:48:12.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:48:12.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:48:12.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:48:12.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 15 op/s 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[24430]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: mgrmap e36: x(active, since 8s), standbys: y 2024-05-15T15:48:12.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 15 op/s 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:12.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:12 smithi043 ceph-mon[29127]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:48:13.738 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:13 smithi053 ceph-mon[25205]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T15:48:13.738 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:13 smithi053 ceph-mon[25205]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:48:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:13 smithi043 ceph-mon[24430]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T15:48:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:13 smithi043 ceph-mon[24430]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:48:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:13 smithi043 ceph-mon[29127]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T15:48:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:13 smithi043 ceph-mon[29127]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:48:14.788 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:14 smithi053 ceph-mon[25205]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:48:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:14 smithi043 ceph-mon[24430]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:48:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:14 smithi043 ceph-mon[29127]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:48:15.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 systemd[1]: Stopping Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:48:15.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 bash[56714]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus.a" found: no such container 2024-05-15T15:48:16.180 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.916Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-15T15:48:16.180 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.916Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-15T15:48:16.180 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.916Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-15T15:48:16.180 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.916Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-15T15:48:16.180 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.916Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-15T15:48:16.181 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.916Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-15T15:48:16.181 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.916Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-15T15:48:16.181 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.916Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-15T15:48:16.181 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.916Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-15T15:48:16.181 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.917Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-15T15:48:16.181 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.917Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-15T15:48:16.181 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[53831]: ts=2024-05-15T15:48:15.917Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-15T15:48:16.181 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:15 smithi053 podman[56725]: 2024-05-15 15:48:15.926554247 +0000 UTC m=+0.207858891 container died 4abe92292f7f3de121ed894126e164b5d903e4169ae90e94b75fabd14e6702d0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:48:16.580 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:16 smithi053 podman[56725]: 2024-05-15 15:48:16.277566794 +0000 UTC m=+0.558871436 container cleanup 4abe92292f7f3de121ed894126e164b5d903e4169ae90e94b75fabd14e6702d0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:48:16.580 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:16 smithi053 bash[56725]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a 2024-05-15T15:48:16.580 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:16 smithi053 bash[56752]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus.a" found: no such container 2024-05-15T15:48:16.839 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:16 smithi053 podman[56739]: 2024-05-15 15:48:16.578956053 +0000 UTC m=+0.649459997 container remove 4abe92292f7f3de121ed894126e164b5d903e4169ae90e94b75fabd14e6702d0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:48:16.839 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:16 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-15T15:48:16.839 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:16 smithi053 systemd[1]: Stopped Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:48:16.839 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:16 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Consumed 1.431s CPU time. 2024-05-15T15:48:16.840 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:16 smithi053 systemd[1]: Starting Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:48:16.840 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:16 smithi053 podman[56816]: 2024-05-15 15:48:16.728572972 +0000 UTC m=+0.019361909 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-15T15:48:16.840 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:16 smithi053 podman[56816]: 2024-05-15 15:48:16.838197961 +0000 UTC m=+0.128986885 container create 6e292811fe04abe871cc665213d5e151b67d268c805111d5f8552238908cccc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 podman[56816]: 2024-05-15 15:48:17.032138772 +0000 UTC m=+0.322927700 container init 6e292811fe04abe871cc665213d5e151b67d268c805111d5f8552238908cccc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 podman[56816]: 2024-05-15 15:48:17.034999853 +0000 UTC m=+0.325788778 container start 6e292811fe04abe871cc665213d5e151b67d268c805111d5f8552238908cccc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.060Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.060Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.060Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.060Z caller=main.go:518 level=info host_details="(Linux 5.14.0-446.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024 x86_64 smithi053 (none))" 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.060Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.060Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.062Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.062Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.063Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.065Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-15T15:48:17.100 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.065Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.913µs 2024-05-15T15:48:17.101 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.065Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-15T15:48:17.101 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 bash[56816]: 6e292811fe04abe871cc665213d5e151b67d268c805111d5f8552238908cccc5 2024-05-15T15:48:17.380 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 systemd[1]: Started Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:48:17.380 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.295Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-05-15T15:48:17.777 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:17 smithi053 ceph-mon[25205]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:48:17.777 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:17 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:17.777 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:17 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:17.777 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:17 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:48:17.777 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:17 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:48:17.777 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.559Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-05-15T15:48:17.777 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.775Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-05-15T15:48:17.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:17 smithi043 ceph-mon[24430]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:48:17.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:17 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:17.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:17 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:17.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:17 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:48:17.801 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:17 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:48:17.801 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:17 smithi043 ceph-mon[29127]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:48:17.802 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:17 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:17.802 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:17 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:17.802 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:17 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:48:17.802 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:17 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:48:18.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.822Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-05-15T15:48:18.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.822Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-05-15T15:48:18.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.822Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=59.619µs wal_replay_duration=756.582454ms total_replay_duration=756.663336ms 2024-05-15T15:48:18.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.828Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-15T15:48:18.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.828Z caller=main.go:947 level=info msg="TSDB started" 2024-05-15T15:48:18.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.828Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-15T15:48:18.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.851Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.59887ms db_storage=816ns remote_storage=1.37µs web_handler=393ns query_engine=862ns scrape=401.79µs scrape_sd=103.694µs notify=16.224µs notify_sd=10.512µs rules=21.715233ms 2024-05-15T15:48:18.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:48:17 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:48:17.851Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-15T15:48:18.733 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:18 smithi053 ceph-mon[25205]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:48:18.753 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:18 smithi043 ceph-mon[24430]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:48:18.753 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:18 smithi043 ceph-mon[29127]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:48:19.728 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:19 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:19.728 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:19 smithi053 ceph-mon[25205]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:48:19.729 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:19 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:19.729 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:19 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:19 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:19 smithi043 ceph-mon[24430]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:48:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:19 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:19 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:19 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:19 smithi043 ceph-mon[29127]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:48:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:19 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:19.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:19 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:21 smithi043 ceph-mon[24430]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:48:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:21 smithi043 ceph-mon[29127]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:48:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:21 smithi053 ceph-mon[25205]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:48:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:48:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:48:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[24430]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:48:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[24430]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:22.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:22.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:22.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:48:22.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:48:22.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[29127]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:48:22.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[29127]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:22.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:22 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:22 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:22 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:22 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:48:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:22 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:48:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:22 smithi053 ceph-mon[25205]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:48:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:22 smithi053 ceph-mon[25205]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:22 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:48:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:23 smithi043 ceph-mon[24430]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:23 smithi043 ceph-mon[29127]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:23 smithi053 ceph-mon[25205]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:24.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:48:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:48:24] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-05-15T15:48:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:26 smithi043 ceph-mon[24430]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:26 smithi043 ceph-mon[29127]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:26 smithi053 ceph-mon[25205]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:27 smithi043 ceph-mon[24430]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:27.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:27 smithi043 ceph-mon[29127]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:27 smithi053 ceph-mon[25205]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:48:28.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:28 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:28.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:28 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:28 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:29 smithi043 ceph-mon[24430]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 553 B/s rd, 0 op/s 2024-05-15T15:48:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:29 smithi043 ceph-mon[29127]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 553 B/s rd, 0 op/s 2024-05-15T15:48:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:29 smithi053 ceph-mon[25205]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 553 B/s rd, 0 op/s 2024-05-15T15:48:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:31 smithi043 ceph-mon[24430]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:48:31.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:31 smithi043 ceph-mon[29127]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:48:31.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:31 smithi053 ceph-mon[25205]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:48:33.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:33 smithi043 ceph-mon[24430]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:33 smithi043 ceph-mon[29127]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:33 smithi053 ceph-mon[25205]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:34 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:34.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:34 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:34 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:34.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:48:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:48:34] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-05-15T15:48:36.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:36 smithi043 ceph-mon[24430]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:36.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:36 smithi043 ceph-mon[29127]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:36 smithi053 ceph-mon[25205]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:37 smithi043 ceph-mon[29127]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:37.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:37 smithi043 ceph-mon[24430]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:37 smithi053 ceph-mon[25205]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:38.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:38 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:38 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:38 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:39 smithi043 ceph-mon[24430]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:39 smithi043 ceph-mon[29127]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:39 smithi053 ceph-mon[25205]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:41 smithi043 ceph-mon[24430]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:41 smithi043 ceph-mon[29127]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:41 smithi053 ceph-mon[25205]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:43 smithi043 ceph-mon[24430]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:43 smithi043 ceph-mon[29127]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:43 smithi053 ceph-mon[25205]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:44.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:48:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:48:44] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-05-15T15:48:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:46 smithi043 ceph-mon[24430]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:46.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:46 smithi043 ceph-mon[29127]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:46.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:46 smithi053 ceph-mon[25205]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:47.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:47 smithi043 ceph-mon[24430]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:47 smithi043 ceph-mon[29127]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:47 smithi053 ceph-mon[25205]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:48 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:48.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:48 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:48 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:49 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:49 smithi043 ceph-mon[24430]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:49 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:49 smithi043 ceph-mon[29127]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:49 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:48:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:49 smithi053 ceph-mon[25205]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:51.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:51 smithi043 ceph-mon[24430]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:51 smithi043 ceph-mon[29127]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:51 smithi053 ceph-mon[25205]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:53 smithi043 ceph-mon[24430]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:53 smithi043 ceph-mon[29127]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:53 smithi053 ceph-mon[25205]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:54.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:48:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:48:54] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-05-15T15:48:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:56 smithi043 ceph-mon[24430]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:56.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:56 smithi043 ceph-mon[29127]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:56.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:56 smithi053 ceph-mon[25205]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:48:57.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:57 smithi043 ceph-mon[24430]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:57 smithi043 ceph-mon[29127]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:57 smithi053 ceph-mon[25205]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:58.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:58 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:58.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:58 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:58.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:58 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:48:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:48:59 smithi043 ceph-mon[24430]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:48:59 smithi043 ceph-mon[29127]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:48:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:48:59 smithi053 ceph-mon[25205]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:01 smithi043 ceph-mon[29127]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:01.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:01 smithi043 ceph-mon[24430]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:01 smithi053 ceph-mon[25205]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:03 smithi043 ceph-mon[29127]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:03.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:03 smithi043 ceph-mon[24430]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:03 smithi053 ceph-mon[25205]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:04.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:49:04 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:49:04] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-05-15T15:49:06.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:06 smithi043 ceph-mon[24430]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:06.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:06 smithi043 ceph-mon[29127]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:06.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:06 smithi053 ceph-mon[25205]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:07.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:07 smithi043 ceph-mon[24430]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:07.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:07 smithi043 ceph-mon[29127]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:07 smithi053 ceph-mon[25205]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:08 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:08 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:08 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:09 smithi043 ceph-mon[24430]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:09 smithi043 ceph-mon[29127]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:09 smithi053 ceph-mon[25205]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:11 smithi043 ceph-mon[24430]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:11 smithi043 ceph-mon[29127]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:11 smithi053 ceph-mon[25205]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:13 smithi043 ceph-mon[24430]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:13 smithi043 ceph-mon[29127]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:13 smithi053 ceph-mon[25205]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:14.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:49:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:49:14] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-05-15T15:49:16.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:16 smithi043 ceph-mon[24430]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:16.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:16 smithi043 ceph-mon[29127]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:16.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:16 smithi053 ceph-mon[25205]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:17 smithi043 ceph-mon[24430]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:17 smithi043 ceph-mon[29127]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:17 smithi053 ceph-mon[25205]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:18.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:18 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:18.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:18 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:18 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:19 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:19 smithi043 ceph-mon[24430]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:19 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:19 smithi043 ceph-mon[29127]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:19 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:19 smithi053 ceph-mon[25205]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:21 smithi043 ceph-mon[24430]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:21 smithi043 ceph-mon[29127]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:21 smithi053 ceph-mon[25205]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:49:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:49:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:49:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[29127]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-05-15T15:49:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[29127]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:49:22.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:49:22.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:49:22.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:49:22.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[24430]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-05-15T15:49:22.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[24430]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:22.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:22 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:49:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:22 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:49:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:22 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:49:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:22 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:49:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:22 smithi053 ceph-mon[25205]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 947 B/s rd, 0 op/s 2024-05-15T15:49:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:22 smithi053 ceph-mon[25205]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:22 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:49:24.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:49:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:49:24] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-05-15T15:49:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:25 smithi053 ceph-mon[25205]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-05-15T15:49:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:25 smithi043 ceph-mon[24430]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-05-15T15:49:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:25 smithi043 ceph-mon[29127]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 581 B/s rd, 0 op/s 2024-05-15T15:49:27.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:27 smithi043 ceph-mon[24430]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:27.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:27 smithi043 ceph-mon[29127]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:27 smithi053 ceph-mon[25205]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:29 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:29 smithi043 ceph-mon[24430]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:29 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:29 smithi043 ceph-mon[29127]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:29 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:29 smithi053 ceph-mon[25205]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:31.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:31 smithi043 ceph-mon[29127]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:31 smithi043 ceph-mon[24430]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:31.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:31 smithi053 ceph-mon[25205]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:49:33.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:33 smithi043 ceph-mon[24430]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:49:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:33 smithi043 ceph-mon[29127]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:49:33.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:33 smithi053 ceph-mon[25205]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:49:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:34 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:34.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:34 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:34 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:34.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:49:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:49:34] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-05-15T15:49:35.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:35 smithi043 ceph-mon[24430]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:35.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:35 smithi043 ceph-mon[29127]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:35.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:35 smithi053 ceph-mon[25205]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:37.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:37 smithi043 ceph-mon[24430]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:37.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:37 smithi043 ceph-mon[29127]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:37 smithi053 ceph-mon[25205]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:39 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:39 smithi043 ceph-mon[24430]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:39 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:39 smithi043 ceph-mon[29127]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:39 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:39 smithi053 ceph-mon[25205]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:41 smithi043 ceph-mon[24430]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:41 smithi043 ceph-mon[29127]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:41 smithi053 ceph-mon[25205]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:43.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:43 smithi043 ceph-mon[29127]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:43 smithi043 ceph-mon[24430]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:43 smithi053 ceph-mon[25205]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:44.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:49:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:49:44] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-05-15T15:49:45.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:45 smithi043 ceph-mon[24430]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:45.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:45 smithi043 ceph-mon[29127]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:45.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:45 smithi053 ceph-mon[25205]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:47.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:47 smithi043 ceph-mon[24430]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:47 smithi043 ceph-mon[29127]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:47 smithi053 ceph-mon[25205]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:49 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:49 smithi043 ceph-mon[24430]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:49 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:49 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:49 smithi043 ceph-mon[29127]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:49 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:49 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:49 smithi053 ceph-mon[25205]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:49 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:49:51.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:51 smithi043 ceph-mon[24430]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:51 smithi043 ceph-mon[29127]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:51 smithi053 ceph-mon[25205]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:53 smithi043 ceph-mon[24430]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:53 smithi043 ceph-mon[29127]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:53 smithi053 ceph-mon[25205]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:54.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:49:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:49:54] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-05-15T15:49:55.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:55 smithi053 ceph-mon[25205]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:55 smithi043 ceph-mon[24430]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:55 smithi043 ceph-mon[29127]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:57.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:57 smithi043 ceph-mon[24430]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:57 smithi043 ceph-mon[29127]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:57 smithi053 ceph-mon[25205]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:49:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:59 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:49:59 smithi043 ceph-mon[24430]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:59 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:49:59 smithi043 ceph-mon[29127]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:49:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:59 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:49:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:49:59 smithi053 ceph-mon[25205]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:00 smithi043 ceph-mon[24430]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:50:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:00 smithi043 ceph-mon[24430]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:50:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:00 smithi043 ceph-mon[24430]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in error state 2024-05-15T15:50:00.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:00 smithi043 ceph-mon[29127]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:50:00.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:00 smithi043 ceph-mon[29127]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:50:00.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:00 smithi043 ceph-mon[29127]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in error state 2024-05-15T15:50:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:00 smithi053 ceph-mon[25205]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:50:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:00 smithi053 ceph-mon[25205]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:50:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:00 smithi053 ceph-mon[25205]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in error state 2024-05-15T15:50:01.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:01 smithi043 ceph-mon[24430]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:01 smithi043 ceph-mon[29127]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:01 smithi053 ceph-mon[25205]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:03.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:03 smithi043 ceph-mon[24430]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:03 smithi043 ceph-mon[29127]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:03 smithi053 ceph-mon[25205]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:04.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:50:04 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:50:04] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-05-15T15:50:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:05 smithi043 ceph-mon[24430]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:05 smithi043 ceph-mon[29127]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:05 smithi053 ceph-mon[25205]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:06.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:06 smithi043 ceph-mon[24430]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:06.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:06 smithi043 ceph-mon[29127]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:06.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:06 smithi053 ceph-mon[25205]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:09.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:09 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:09.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:09 smithi053 ceph-mon[25205]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:09 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:09 smithi043 ceph-mon[24430]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:09 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:09 smithi043 ceph-mon[29127]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:11 smithi043 ceph-mon[24430]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:11.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:11 smithi043 ceph-mon[29127]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:11 smithi053 ceph-mon[25205]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:13 smithi043 ceph-mon[24430]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:13 smithi043 ceph-mon[29127]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:13 smithi053 ceph-mon[25205]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:14.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:50:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:50:14] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-05-15T15:50:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:15 smithi043 ceph-mon[24430]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:15 smithi043 ceph-mon[29127]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:15 smithi053 ceph-mon[25205]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:17 smithi043 ceph-mon[24430]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:17 smithi043 ceph-mon[29127]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:17 smithi053 ceph-mon[25205]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:19 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:19 smithi043 ceph-mon[29127]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:19 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:19.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:19 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:19.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:19 smithi043 ceph-mon[24430]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:19.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:19 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:19 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:19 smithi053 ceph-mon[25205]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:19 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:21 smithi043 ceph-mon[24430]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:21 smithi043 ceph-mon[29127]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:21 smithi053 ceph-mon[25205]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:22 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:50:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:22 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:50:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:22 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:50:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:23 smithi043 ceph-mon[24430]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:23 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:50:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:23 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:50:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:23 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:50:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:23 smithi043 ceph-mon[29127]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:23 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:50:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:23 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:50:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:23 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:50:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:23 smithi053 ceph-mon[25205]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:23 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:50:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:23 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:50:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:23 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:50:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:24 smithi043 ceph-mon[24430]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-05-15T15:50:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:24 smithi043 ceph-mon[24430]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:24.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:24 smithi043 ceph-mon[29127]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-05-15T15:50:24.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:24 smithi043 ceph-mon[29127]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:24.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:24 smithi053 ceph-mon[25205]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-05-15T15:50:24.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:24 smithi053 ceph-mon[25205]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:24.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:50:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:50:24] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-05-15T15:50:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:26 smithi043 ceph-mon[24430]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:26 smithi043 ceph-mon[29127]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:26 smithi053 ceph-mon[25205]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:28.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:28 smithi043 ceph-mon[24430]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:28.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:28 smithi043 ceph-mon[29127]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:28 smithi053 ceph-mon[25205]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:29 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:29 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:29 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:30 smithi043 ceph-mon[24430]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-05-15T15:50:30.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:30 smithi043 ceph-mon[29127]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-05-15T15:50:30.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:30 smithi053 ceph-mon[25205]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-05-15T15:50:32.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:32 smithi043 ceph-mon[24430]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:32.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:32 smithi043 ceph-mon[29127]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:32.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:32 smithi053 ceph-mon[25205]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:50:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:34 smithi043 ceph-mon[24430]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:50:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:34 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:34.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:34 smithi043 ceph-mon[29127]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:50:34.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:34 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:34 smithi053 ceph-mon[25205]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:50:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:34 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:34.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:50:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:50:34] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-05-15T15:50:36.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:36 smithi043 ceph-mon[24430]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:36.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:36 smithi043 ceph-mon[29127]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:36 smithi053 ceph-mon[25205]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:38.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:38 smithi043 ceph-mon[24430]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:38 smithi043 ceph-mon[29127]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:38.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:38 smithi053 ceph-mon[25205]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:39 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:39 smithi043 ceph-mon[24430]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:39 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:39 smithi043 ceph-mon[29127]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:39 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:39 smithi053 ceph-mon[25205]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:42 smithi043 ceph-mon[24430]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:42 smithi043 ceph-mon[29127]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:42 smithi053 ceph-mon[25205]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:44.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:44 smithi043 ceph-mon[24430]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:44.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:44 smithi043 ceph-mon[29127]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:44.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:44 smithi053 ceph-mon[25205]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:44.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:50:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:50:44] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-05-15T15:50:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:46 smithi043 ceph-mon[24430]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:46.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:46 smithi043 ceph-mon[29127]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:46.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:46 smithi053 ceph-mon[25205]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:48 smithi043 ceph-mon[24430]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:48.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:48 smithi043 ceph-mon[29127]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:48 smithi053 ceph-mon[25205]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:49 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:49 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:49 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:49 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:49 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:49 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:50:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:50 smithi043 ceph-mon[24430]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:50.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:50 smithi043 ceph-mon[29127]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:50 smithi053 ceph-mon[25205]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:52.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:52 smithi043 ceph-mon[29127]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:52 smithi043 ceph-mon[24430]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:52 smithi053 ceph-mon[25205]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:54.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:54 smithi043 ceph-mon[24430]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:54 smithi043 ceph-mon[29127]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:54 smithi053 ceph-mon[25205]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:54.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:50:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:50:54] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-05-15T15:50:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:56 smithi043 ceph-mon[24430]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:56.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:56 smithi043 ceph-mon[29127]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:56.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:56 smithi053 ceph-mon[25205]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:50:58.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:58 smithi043 ceph-mon[24430]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:58.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:58 smithi043 ceph-mon[29127]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:58.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:58 smithi053 ceph-mon[25205]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:50:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:50:59 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:50:59 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:50:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:50:59 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:00 smithi043 ceph-mon[24430]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:00.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:00 smithi043 ceph-mon[29127]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:00 smithi053 ceph-mon[25205]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:02.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:02 smithi043 ceph-mon[24430]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:02.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:02 smithi043 ceph-mon[29127]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:02.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:02 smithi053 ceph-mon[25205]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:04 smithi043 ceph-mon[24430]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:04 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:04.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:04 smithi043 ceph-mon[29127]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:04.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:04 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:04 smithi053 ceph-mon[25205]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:04.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:04 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:04.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:04 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:51:04] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-05-15T15:51:06.770 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:06 smithi043 ceph-mon[24430]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:06.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:06 smithi043 ceph-mon[29127]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:06.799 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T15:51:06.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:06 smithi053 ceph-mon[25205]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:08.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:08 smithi043 ceph-mon[24430]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:08.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:08 smithi043 ceph-mon[29127]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:08.856 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:51:08.856 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (12m) 2m ago 20m 23.6M - 0.23.0 ba2b418f427c dd5a5b0f3fe5 2024-05-15T15:51:08.856 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (12m) 2m ago 20m 52.0M - 8.3.5 dad864ee21e9 3fece04315b1 2024-05-15T15:51:08.856 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 error 2m ago 19m 54.5M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:51:08.856 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283 running (15m) 2m ago 24m 492M - 19.0.0-2576-gde5210d5 48bc18eb22d8 cfeda1bd207c 2024-05-15T15:51:08.856 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (9m) 2m ago 26m 123M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (26m) 2m ago 26m 68.0M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (25m) 2m ago 25m 55.4M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (25m) 2m ago 25m 50.6M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (12m) 2m ago 20m 15.6M - 1.3.1 1dbe0e931976 083a05fd3833 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (12m) 2m ago 20m 17.1M - 1.3.1 1dbe0e931976 2c6976b8c648 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (24m) 2m ago 24m 53.0M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (24m) 2m ago 24m 56.7M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (23m) 2m ago 23m 49.8M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (23m) 2m ago 23m 52.2M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (22m) 2m ago 22m 52.0M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:51:08.857 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (22m) 2m ago 22m 50.5M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:51:08.858 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (22m) 2m ago 22m 53.5M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:51:08.858 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (21m) 2m ago 21m 55.3M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:51:08.858 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (2m) 2m ago 21m 43.9M - 2.33.4 514e6a882f6e 6e292811fe04 2024-05-15T15:51:08.858 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (19m) 2m ago 19m 75.0M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:51:08.858 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (19m) 2m ago 19m 74.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:51:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:08 smithi053 ceph-mon[25205]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:09.560 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:09 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:09.560 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:09 smithi043 ceph-mon[24430]: from='client.15303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:09.560 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:09 smithi043 ceph-mon[24430]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:09.560 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:09 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:09.560 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:09 smithi043 ceph-mon[29127]: from='client.15303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:09.561 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:09 smithi043 ceph-mon[29127]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:09.587 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ls' 2024-05-15T15:51:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:09 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:09 smithi053 ceph-mon[25205]: from='client.15303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:09 smithi053 ceph-mon[25205]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:11.612 INFO:teuthology.orchestra.run.smithi043.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-15T15:51:11.612 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi043=a;count:1 2024-05-15T15:51:11.612 INFO:teuthology.orchestra.run.smithi043.stdout:grafana ?:3000 1/1 2m ago 21m smithi053=a;count:1 2024-05-15T15:51:11.612 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo ?:5000 0/1 3m ago 19m count:1 2024-05-15T15:51:11.612 INFO:teuthology.orchestra.run.smithi043.stdout:mgr 2/2 3m ago 24m smithi043=y;smithi053=x;count:2 2024-05-15T15:51:11.613 INFO:teuthology.orchestra.run.smithi043.stdout:mon 3/3 3m ago 25m smithi043:172.21.15.43=a;smithi043:[v2:172.21.15.43:3301,v1:172.21.15.43:6790]=c;smithi053:172.21.15.53=b;count:3 2024-05-15T15:51:11.613 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi043=a;smithi053=b;count:2 2024-05-15T15:51:11.613 INFO:teuthology.orchestra.run.smithi043.stdout:osd 8 3m ago - 2024-05-15T15:51:11.613 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus ?:9095 1/1 2m ago 21m smithi053=a;count:1 2024-05-15T15:51:11.613 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-05-15T15:51:12.351 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions' 2024-05-15T15:51:12.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:12 smithi043 ceph-mon[24430]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:12.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:12 smithi043 ceph-mon[29127]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:12.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:12 smithi053 ceph-mon[25205]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:13.748 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:13 smithi043 ceph-mon[24430]: from='client.25216 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:13.749 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:13 smithi043 ceph-mon[29127]: from='client.25216 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:13 smithi053 ceph-mon[25205]: from='client.25216 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:14.545 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:51:14.545 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:51:14.545 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:51:14.545 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:14.545 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:51:14.545 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:51:14.546 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:51:14.547 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:51:14.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:14 smithi053 ceph-mon[25205]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:14.782 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:51:14] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-05-15T15:51:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:14 smithi043 ceph-mon[24430]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:14 smithi043 ceph-mon[29127]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:15.143 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-05-15T15:51:15.786 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:15 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/863727043' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:15.786 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:15 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/863727043' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:15 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/863727043' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:16.738 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:16 smithi043 ceph-mon[24430]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:16.739 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:16 smithi043 ceph-mon[29127]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:16.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:16 smithi053 ceph-mon[25205]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:18.549 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:18 smithi043 ceph-mon[24430]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:18.549 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:18 smithi043 ceph-mon[24430]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:18.549 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:18 smithi043 ceph-mon[29127]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:18.550 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:18 smithi043 ceph-mon[29127]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:18 smithi053 ceph-mon[25205]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:18 smithi053 ceph-mon[25205]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:19.264 INFO:teuthology.orchestra.run.smithi043.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:51:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:51:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:51:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:51:19.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:19.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:19.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:51:19.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:19.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:19.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:51:19.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:19 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:51:19.875 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-15T15:51:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:19 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:19 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:19 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:51:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:19 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:19 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:19 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:51:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:19 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:51:20.805 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:20 smithi043 ceph-mon[29127]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:20.805 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:20 smithi043 ceph-mon[29127]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:51:20.805 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:20 smithi043 ceph-mon[29127]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-05-15T15:51:20.805 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:20 smithi043 ceph-mon[29127]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-05-15T15:51:20.805 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:20 smithi043 ceph-mon[29127]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:51:20.806 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:20 smithi043 ceph-mon[24430]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:20.806 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:20 smithi043 ceph-mon[24430]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:51:20.806 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:20 smithi043 ceph-mon[24430]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-05-15T15:51:20.806 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:20 smithi043 ceph-mon[24430]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-05-15T15:51:20.806 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:20 smithi043 ceph-mon[24430]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:51:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:20 smithi053 ceph-mon[25205]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:51:20.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:20 smithi053 ceph-mon[25205]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:51:20.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:20 smithi053 ceph-mon[25205]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-05-15T15:51:20.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:20 smithi053 ceph-mon[25205]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-05-15T15:51:20.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:20 smithi053 ceph-mon[25205]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:51:22.117 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:51:22.805 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:51:22.805 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (12m) 3m ago 20m 23.6M - 0.23.0 ba2b418f427c dd5a5b0f3fe5 2024-05-15T15:51:22.805 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (12m) 3m ago 20m 52.0M - 8.3.5 dad864ee21e9 3fece04315b1 2024-05-15T15:51:22.805 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 error 3m ago 19m 54.5M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:51:22.805 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283 running (15m) 3m ago 25m 492M - 19.0.0-2576-gde5210d5 48bc18eb22d8 cfeda1bd207c 2024-05-15T15:51:22.805 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (9m) 3m ago 26m 123M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:51:22.805 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (26m) 3m ago 26m 68.0M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:51:22.805 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (25m) 3m ago 25m 55.4M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (25m) 3m ago 25m 50.6M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (12m) 3m ago 21m 15.6M - 1.3.1 1dbe0e931976 083a05fd3833 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (12m) 3m ago 21m 17.1M - 1.3.1 1dbe0e931976 2c6976b8c648 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (24m) 3m ago 24m 53.0M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (24m) 3m ago 24m 56.7M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (23m) 3m ago 23m 49.8M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (23m) 3m ago 23m 52.2M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (23m) 3m ago 23m 52.0M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (22m) 3m ago 22m 50.5M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (22m) 3m ago 22m 53.5M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (22m) 3m ago 22m 55.3M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (3m) 3m ago 21m 43.9M - 2.33.4 514e6a882f6e 6e292811fe04 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (19m) 3m ago 19m 75.0M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:51:22.806 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (19m) 3m ago 19m 74.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:51:22.807 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:22 smithi043 ceph-mon[24430]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:22.807 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:22 smithi043 ceph-mon[29127]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:22 smithi053 ceph-mon[25205]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:51:23.230 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:51:23.230 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:51:23.230 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:51:23.230 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:23.230 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:51:23.230 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:51:23.230 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:51:23.231 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:51:23.582 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:51:23.582 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T15:51:23.582 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:51:23.582 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-15T15:51:23.582 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [ 2024-05-15T15:51:23.582 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr" 2024-05-15T15:51:23.582 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T15:51:23.582 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "2/2 daemons upgraded", 2024-05-15T15:51:23.582 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 image", 2024-05-15T15:51:23.582 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:51:23.583 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:51:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[29127]: from='client.25234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[29127]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:51:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[29127]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-15T15:51:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-15T15:51:23.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[29127]: osdmap e106: 8 total, 8 up, 8 in 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[29127]: Standby manager daemon x started 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3351246603' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[24430]: from='client.25234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[24430]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[24430]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[24430]: osdmap e106: 8 total, 8 up, 8 in 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[24430]: Standby manager daemon x started 2024-05-15T15:51:23.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:23 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3351246603' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:23 smithi053 ceph-mon[25205]: from='client.25234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:23 smithi053 ceph-mon[25205]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:23 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' 2024-05-15T15:51:23.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:23 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:23.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:23 smithi053 ceph-mon[25205]: from='mgr.24968 172.21.15.53:0/779110772' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-15T15:51:23.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:23 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-05-15T15:51:23.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:23 smithi053 ceph-mon[25205]: osdmap e106: 8 total, 8 up, 8 in 2024-05-15T15:51:23.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:23 smithi053 ceph-mon[25205]: Standby manager daemon x started 2024-05-15T15:51:23.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:23 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3351246603' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:24.325 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:23.938+0000 7fe29fa21700 -1 mgr handle_mgr_map I was active but no longer am 2024-05-15T15:51:24.325 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ignoring --setuser ceph since I am not root 2024-05-15T15:51:24.325 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:23 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ignoring --setgroup ceph since I am not root 2024-05-15T15:51:24.325 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:23 smithi053 ceph-mgr[45196]: -- 172.21.15.53:0/3316375613 <== mon.2 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ed719fc5a0 con 0x55ed719d0c00 2024-05-15T15:51:24.325 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:24.126+0000 7f48b478f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:51:24.325 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:24.322+0000 7f48b478f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:51:24.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:51:23] ENGINE Bus STOPPING 2024-05-15T15:51:24.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:51:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-15T15:51:24.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:51:23] ENGINE Bus STOPPED 2024-05-15T15:51:24.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:51:24] ENGINE Bus STARTING 2024-05-15T15:51:24.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:51:24] ENGINE Serving on http://:::9283 2024-05-15T15:51:24.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:51:24] ENGINE Bus STARTED 2024-05-15T15:51:24.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:24.434+0000 7f48b478f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:51:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='client.15339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:24.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: Failing over to other MGR 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.24968 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: mgrmap e37: y(active, starting, since 0.997036s), standbys: x 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:51:24.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: Manager daemon y is now available 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:51:24.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:51:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='client.15339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: Failing over to other MGR 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.24968 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: mgrmap e37: y(active, starting, since 0.997036s), standbys: x 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:51:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: Manager daemon y is now available 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:51:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:51:24.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='client.15339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:24.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:51:24.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:51:24.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-15T15:51:24.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: Upgrade: Need to upgrade myself (mgr.x) 2024-05-15T15:51:24.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: Failing over to other MGR 2024-05-15T15:51:24.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-05-15T15:51:24.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.24968 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-05-15T15:51:24.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: mgrmap e37: y(active, starting, since 0.997036s), standbys: x 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:51:24.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:51:24.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:51:24.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: Manager daemon y is now available 2024-05-15T15:51:24.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:24.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:24.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:51:24.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:51:25.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:24.770+0000 7f48b478f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:51:25.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:24.873+0000 7f48b478f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:51:25.778 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:25 smithi053 ceph-mon[25205]: [15/May/2024:15:51:24] ENGINE Bus STARTING 2024-05-15T15:51:25.778 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:25 smithi053 ceph-mon[25205]: mgrmap e38: y(active, since 2s), standbys: x 2024-05-15T15:51:25.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:25 smithi043 ceph-mon[29127]: [15/May/2024:15:51:24] ENGINE Bus STARTING 2024-05-15T15:51:25.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:25 smithi043 ceph-mon[29127]: mgrmap e38: y(active, since 2s), standbys: x 2024-05-15T15:51:25.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:25 smithi043 ceph-mon[24430]: [15/May/2024:15:51:24] ENGINE Bus STARTING 2024-05-15T15:51:25.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:25 smithi043 ceph-mon[24430]: mgrmap e38: y(active, since 2s), standbys: x 2024-05-15T15:51:26.512 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:26.198+0000 7f48b478f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:51:26.512 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:26.395+0000 7f48b478f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:51:26.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[24430]: [15/May/2024:15:51:24] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:51:26.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[24430]: [15/May/2024:15:51:24] ENGINE Serving on http://172.21.15.43:8765 2024-05-15T15:51:26.796 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[24430]: [15/May/2024:15:51:24] ENGINE Bus STARTED 2024-05-15T15:51:26.796 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[24430]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:26.796 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[24430]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-15T15:51:26.796 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[24430]: Cluster is now healthy 2024-05-15T15:51:26.796 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[24430]: mgrmap e39: y(active, since 3s), standbys: x 2024-05-15T15:51:26.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[29127]: [15/May/2024:15:51:24] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:51:26.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[29127]: [15/May/2024:15:51:24] ENGINE Serving on http://172.21.15.43:8765 2024-05-15T15:51:26.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[29127]: [15/May/2024:15:51:24] ENGINE Bus STARTED 2024-05-15T15:51:26.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[29127]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:26.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[29127]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-15T15:51:26.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[29127]: Cluster is now healthy 2024-05-15T15:51:26.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:26 smithi043 ceph-mon[29127]: mgrmap e39: y(active, since 3s), standbys: x 2024-05-15T15:51:26.809 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:26 smithi053 ceph-mon[25205]: [15/May/2024:15:51:24] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:51:26.809 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:26 smithi053 ceph-mon[25205]: [15/May/2024:15:51:24] ENGINE Serving on http://172.21.15.43:8765 2024-05-15T15:51:26.809 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:26 smithi053 ceph-mon[25205]: [15/May/2024:15:51:24] ENGINE Bus STARTED 2024-05-15T15:51:26.809 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:26 smithi053 ceph-mon[25205]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:26.809 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:26 smithi053 ceph-mon[25205]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-15T15:51:26.809 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:26 smithi053 ceph-mon[25205]: Cluster is now healthy 2024-05-15T15:51:26.810 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:26 smithi053 ceph-mon[25205]: mgrmap e39: y(active, since 3s), standbys: x 2024-05-15T15:51:26.810 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:26.577+0000 7f48b478f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:51:27.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:26.888+0000 7f48b478f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:51:27.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:26.987+0000 7f48b478f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:51:27.497 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:27.218+0000 7f48b478f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:51:27.497 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:27.426+0000 7f48b478f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:51:27.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:27 smithi043 ceph-mon[24430]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:27.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:27 smithi043 ceph-mon[29127]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:27 smithi053 ceph-mon[25205]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:28.339 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:28.076+0000 7f48b478f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:51:28.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:28 smithi053 ceph-mon[25205]: mgrmap e40: y(active, since 4s), standbys: x 2024-05-15T15:51:28.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:28.532+0000 7f48b478f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:51:28.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:28 smithi043 ceph-mon[24430]: mgrmap e40: y(active, since 4s), standbys: x 2024-05-15T15:51:28.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:28 smithi043 ceph-mon[29127]: mgrmap e40: y(active, since 4s), standbys: x 2024-05-15T15:51:29.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:28.637+0000 7f48b478f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:51:29.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:28.862+0000 7f48b478f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-15T15:51:29.810 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:29 smithi053 ceph-mon[25205]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:29.811 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:29 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:29.811 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:29 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.811 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:29 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.811 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:29 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.811 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:29 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.811 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:29.432+0000 7f48b478f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:51:29.811 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:29.537+0000 7f48b478f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:51:29.811 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:29.807+0000 7f48b478f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:51:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[24430]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[29127]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:29.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:29.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:29.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:29 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:30.069 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:29.958+0000 7f48b478f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:51:30.069 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:30.066+0000 7f48b478f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:51:30.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:30.201+0000 7f48b478f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:51:30.842 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:30.537+0000 7f48b478f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:51:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:51:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:51:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:51:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:51:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:51:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:51:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:30 smithi053 ceph-mon[25205]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:51:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:51:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:51:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:31.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:51:31.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:51:31.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:51:31.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:51:31.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:51:31.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:51:31.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:51:31.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:30 smithi043 ceph-mon[24430]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:51:31.541 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:31.197+0000 7f48b478f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:51:31.541 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:31.431+0000 7f48b478f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:51:31.541 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:51:31] ENGINE Bus STARTING 2024-05-15T15:51:31.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: CherryPy Checker: 2024-05-15T15:51:31.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: The Application mounted at '' has an empty config. 2024-05-15T15:51:31.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: 2024-05-15T15:51:31.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:51:31] ENGINE Serving on http://:::9283 2024-05-15T15:51:31.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: [15/May/2024:15:51:31] ENGINE Bus STARTED 2024-05-15T15:51:31.977 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:51:31.977 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:51:31.977 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.977 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.977 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.977 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.977 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 20 op/s 2024-05-15T15:51:31.977 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: Standby manager daemon x restarted 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: Standby manager daemon x started 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:51:31.978 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 20 op/s 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: Standby manager daemon x restarted 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: Standby manager daemon x started 2024-05-15T15:51:31.979 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:51:31.980 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:51:31.980 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:51:31.980 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:31 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:51:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 20 op/s 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:51:32.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: Standby manager daemon x restarted 2024-05-15T15:51:32.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: Standby manager daemon x started 2024-05-15T15:51:32.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:51:32.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:51:32.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:51:32.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:31 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/577581368' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:51:32.929 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:51:32.748+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:51:32.930 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:32.931 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:32.931 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:32.931 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:32.931 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:32.931 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:32.931 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:32.931 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:51:32.932 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:51:32.933 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:32.934 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:33.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:32 smithi043 ceph-mon[24430]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T15:51:33.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:32 smithi043 ceph-mon[24430]: mgrmap e41: y(active, since 9s), standbys: x 2024-05-15T15:51:33.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:32 smithi043 ceph-mon[29127]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T15:51:33.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:32 smithi043 ceph-mon[29127]: mgrmap e41: y(active, since 9s), standbys: x 2024-05-15T15:51:33.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:32 smithi053 ceph-mon[25205]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T15:51:33.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:32 smithi053 ceph-mon[25205]: mgrmap e41: y(active, since 9s), standbys: x 2024-05-15T15:51:34.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:33 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:34.336 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:34.336 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:34.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:34.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:34.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:34.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:34.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:34.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:34.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:51:34.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:51:34.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:51:34.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:51:34.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:51:34.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:51:34.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:51:34.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:51:34.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:51:34.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:34.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:34.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:34.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:34.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:34.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:51:34.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:34.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:34.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:34.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:33 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:34.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:34.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:51:34.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:34.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:34.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:34.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:51:34.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:34.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:34.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:34.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:34.346 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:34.346 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:34.347 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:34.347 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.347 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:33 smithi043 ceph-mon[24430]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:51:34.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:33 smithi043 ceph-mon[24430]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 15 op/s 2024-05-15T15:51:34.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:33 smithi043 ceph-mon[24430]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:51:34.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:34.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:34.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:34.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.348 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:33 smithi043 ceph-mon[29127]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:51:34.348 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:33 smithi043 ceph-mon[29127]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 15 op/s 2024-05-15T15:51:34.348 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:33 smithi043 ceph-mon[29127]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:33 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:34.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:51:34.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:51:34.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:51:34.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:33 smithi053 ceph-mon[25205]: Reconfiguring prometheus.a (dependencies changed)... 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:33 smithi053 ceph-mon[25205]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 15 op/s 2024-05-15T15:51:34.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:33 smithi053 ceph-mon[25205]: Reconfiguring daemon prometheus.a on smithi053 2024-05-15T15:51:34.859 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[45192]: ::ffff:172.21.15.53 - - [15/May/2024:15:51:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-05-15T15:51:36.574 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:36 smithi053 ceph-mon[25205]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:51:36.574 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 systemd[1]: Stopping Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:51:36.574 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 bash[59585]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus.a" found: no such container 2024-05-15T15:51:36.834 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.572Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-15T15:51:36.834 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.572Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-15T15:51:36.834 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.572Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-15T15:51:36.834 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.572Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-15T15:51:36.835 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.572Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-15T15:51:36.835 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.572Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-15T15:51:36.835 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.572Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-15T15:51:36.835 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.573Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-15T15:51:36.835 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.573Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-15T15:51:36.835 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.573Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-15T15:51:36.836 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.573Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-15T15:51:36.836 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[56831]: ts=2024-05-15T15:51:36.573Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-15T15:51:36.836 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 podman[59597]: 2024-05-15 15:51:36.582832978 +0000 UTC m=+0.118315829 container died 6e292811fe04abe871cc665213d5e151b67d268c805111d5f8552238908cccc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:51:36.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:36 smithi043 ceph-mon[24430]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:51:36.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:36 smithi043 ceph-mon[29127]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:51:37.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 podman[59597]: 2024-05-15 15:51:36.913642155 +0000 UTC m=+0.449125003 container cleanup 6e292811fe04abe871cc665213d5e151b67d268c805111d5f8552238908cccc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:51:37.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:36 smithi053 bash[59597]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a 2024-05-15T15:51:37.459 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 podman[59609]: 2024-05-15 15:51:37.181764776 +0000 UTC m=+0.599572672 container remove 6e292811fe04abe871cc665213d5e151b67d268c805111d5f8552238908cccc5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:51:37.459 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 bash[59625]: Error: no container with name or ID "ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus.a" found: no such container 2024-05-15T15:51:37.459 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-15T15:51:37.459 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 systemd[1]: Stopped Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:51:37.459 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Consumed 1.471s CPU time. 2024-05-15T15:51:37.459 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 systemd[1]: Starting Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:51:37.761 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 podman[59695]: 2024-05-15 15:51:37.457796119 +0000 UTC m=+0.103262863 container create f007cf8f16d2fe21822f8900b9749378a6807286ef718317f2e74ba8e27b4b68 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 podman[59695]: 2024-05-15 15:51:37.373927995 +0000 UTC m=+0.019394719 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 podman[59695]: 2024-05-15 15:51:37.668346791 +0000 UTC m=+0.313813497 container init f007cf8f16d2fe21822f8900b9749378a6807286ef718317f2e74ba8e27b4b68 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 podman[59695]: 2024-05-15 15:51:37.671313176 +0000 UTC m=+0.316779873 container start f007cf8f16d2fe21822f8900b9749378a6807286ef718317f2e74ba8e27b4b68 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.695Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.695Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.695Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.695Z caller=main.go:518 level=info host_details="(Linux 5.14.0-446.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024 x86_64 smithi053 (none))" 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.695Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.695Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.698Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.698Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.699Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-05-15T15:51:37.762 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.700Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-15T15:51:37.763 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.700Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.521µs 2024-05-15T15:51:37.763 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.700Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-15T15:51:37.763 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 bash[59695]: f007cf8f16d2fe21822f8900b9749378a6807286ef718317f2e74ba8e27b4b68 2024-05-15T15:51:38.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 systemd[1]: Started Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:51:38.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:37.923Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-05-15T15:51:38.468 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.179Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-05-15T15:51:38.468 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.404Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-05-15T15:51:38.468 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.466Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-05-15T15:51:38.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:38 smithi053 ceph-mon[25205]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 76 op/s 2024-05-15T15:51:38.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:38.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:38.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:51:38.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:38.782 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.475Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-05-15T15:51:38.782 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.475Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-05-15T15:51:38.782 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.475Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=47.464µs wal_replay_duration=774.521183ms total_replay_duration=774.581802ms 2024-05-15T15:51:38.782 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.479Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-05-15T15:51:38.782 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.479Z caller=main.go:947 level=info msg="TSDB started" 2024-05-15T15:51:38.782 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.479Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-15T15:51:38.782 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.506Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.962977ms db_storage=732ns remote_storage=1.568µs web_handler=385ns query_engine=688ns scrape=596.467µs scrape_sd=126.516µs notify=10.15µs notify_sd=10.645µs rules=25.841481ms 2024-05-15T15:51:38.783 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:51:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:51:38.507Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-05-15T15:51:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:38 smithi043 ceph-mon[29127]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 76 op/s 2024-05-15T15:51:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:51:38.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:38.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:38 smithi043 ceph-mon[24430]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 76 op/s 2024-05-15T15:51:38.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:38.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:38.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:51:38.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:39 smithi053 ceph-mon[25205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:51:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:39 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:39 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:39 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:51:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:39 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:51:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:39 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[24430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:51:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:51:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:51:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[29127]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:51:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:51:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:51:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:39 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:40 smithi043 ceph-mon[24430]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 76 op/s 2024-05-15T15:51:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:40 smithi043 ceph-mon[24430]: Upgrade: Updating mgr.x 2024-05-15T15:51:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:40 smithi043 ceph-mon[24430]: Deploying daemon mgr.x on smithi053 2024-05-15T15:51:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:40 smithi043 ceph-mon[29127]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 76 op/s 2024-05-15T15:51:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:40 smithi043 ceph-mon[29127]: Upgrade: Updating mgr.x 2024-05-15T15:51:40.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:40 smithi043 ceph-mon[29127]: Deploying daemon mgr.x on smithi053 2024-05-15T15:51:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:40 smithi053 ceph-mon[25205]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 76 op/s 2024-05-15T15:51:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:40 smithi053 ceph-mon[25205]: Upgrade: Updating mgr.x 2024-05-15T15:51:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:40 smithi053 ceph-mon[25205]: Deploying daemon mgr.x on smithi053 2024-05-15T15:51:41.512 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:41 smithi053 systemd[1]: Stopping Ceph mgr.x for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:51:41.773 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:41 smithi053 podman[60047]: 2024-05-15 15:51:41.509512117 +0000 UTC m=+0.136469483 container died cfeda1bd207cf7e69dcc383e3e96120ed7dca0fc5974503d3c8f79bca9d884ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2) 2024-05-15T15:51:42.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:41 smithi053 podman[60047]: 2024-05-15 15:51:41.862351394 +0000 UTC m=+0.489308764 container cleanup cfeda1bd207cf7e69dcc383e3e96120ed7dca0fc5974503d3c8f79bca9d884ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d) 2024-05-15T15:51:42.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:41 smithi053 bash[60047]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x 2024-05-15T15:51:42.484 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 podman[60059]: 2024-05-15 15:51:42.130510926 +0000 UTC m=+0.618413685 container remove cfeda1bd207cf7e69dcc383e3e96120ed7dca0fc5974503d3c8f79bca9d884ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-05-15T15:51:42.484 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-05-15T15:51:42.484 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-05-15T15:51:42.484 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 systemd[1]: Stopped Ceph mgr.x for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:51:42.484 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.x.service: Consumed 1min 3.013s CPU time. 2024-05-15T15:51:42.485 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 systemd[1]: Starting Ceph mgr.x for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:51:42.827 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:42 smithi053 ceph-mon[25205]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-05-15T15:51:42.827 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 podman[60193]: 2024-05-15 15:51:42.606055488 +0000 UTC m=+0.018128318 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:51:42.827 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 podman[60193]: 2024-05-15 15:51:42.706591217 +0000 UTC m=+0.118664051 container create 006c00101d675e8c77d581e5086ba26be7c298eff75f065910680f593119c9ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-15T15:51:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:42 smithi043 ceph-mon[24430]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-05-15T15:51:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:42 smithi043 ceph-mon[29127]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-05-15T15:51:43.332 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 podman[60193]: 2024-05-15 15:51:42.967588058 +0000 UTC m=+0.379660873 container init 006c00101d675e8c77d581e5086ba26be7c298eff75f065910680f593119c9ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5) 2024-05-15T15:51:43.332 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 podman[60193]: 2024-05-15 15:51:42.97067499 +0000 UTC m=+0.382747810 container start 006c00101d675e8c77d581e5086ba26be7c298eff75f065910680f593119c9ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240514, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-05-15T15:51:43.332 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:42 smithi053 ceph-mgr[60211]: -- 172.21.15.53:0/2731686042 <== mon.2 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cf779c85a0 con 0x55cf7799d000 2024-05-15T15:51:43.332 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:43 smithi053 bash[60193]: 006c00101d675e8c77d581e5086ba26be7c298eff75f065910680f593119c9ae 2024-05-15T15:51:43.333 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:43 smithi053 systemd[1]: Started Ceph mgr.x for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:51:43.333 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:43.130+0000 7f8b34768200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:51:43.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:43.330+0000 7f8b34768200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:51:43.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:43.441+0000 7f8b34768200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:51:44.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:43.813+0000 7f8b34768200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:51:44.130 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:43.905+0000 7f8b34768200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:51:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:44 smithi043 ceph-mon[24430]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-15T15:51:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:44 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:44 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:44 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:44.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:44 smithi043 ceph-mon[29127]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-15T15:51:44.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:44 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:44.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:44 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:44.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:44 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:44 smithi053 ceph-mon[25205]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-15T15:51:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:44 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:44 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:44 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:45.444 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:45.240+0000 7f8b34768200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:51:45.444 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:45.442+0000 7f8b34768200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:51:45.788 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:45.625+0000 7f8b34768200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:51:46.042 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:45.941+0000 7f8b34768200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:51:46.042 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:46.040+0000 7f8b34768200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:51:46.351 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:46.261+0000 7f8b34768200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:51:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:46 smithi053 ceph-mon[25205]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-15T15:51:46.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:46.471+0000 7f8b34768200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:51:46.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:46 smithi043 ceph-mon[29127]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-15T15:51:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:46 smithi043 ceph-mon[24430]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-05-15T15:51:46.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:51:46] "GET /metrics HTTP/1.1" 200 37978 "" "Prometheus/2.33.4" 2024-05-15T15:51:47.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:47.134+0000 7f8b34768200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:51:47.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:47.600+0000 7f8b34768200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:51:47.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:47.709+0000 7f8b34768200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:51:48.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:47.933+0000 7f8b34768200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-15T15:51:48.598 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:48 smithi043 ceph-mon[24430]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-15T15:51:48.598 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:48.598 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:48.598 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:48 smithi043 ceph-mon[29127]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-15T15:51:48.598 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:48.599 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:48 smithi053 ceph-mon[25205]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-05-15T15:51:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:48 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:48 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:48.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:48.507+0000 7f8b34768200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:51:48.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:48.612+0000 7f8b34768200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:51:49.149 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:48.890+0000 7f8b34768200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:51:49.149 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:49.039+0000 7f8b34768200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:51:49.149 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:49.146+0000 7f8b34768200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:51:49.423 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:49.280+0000 7f8b34768200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:51:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:49 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:49 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:49 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:49 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:49 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:49 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:49 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:49 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:49 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:49.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:49.609+0000 7f8b34768200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:51:50.532 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:50.267+0000 7f8b34768200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:51:50.532 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:50.530+0000 7f8b34768200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:51:50.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:50 smithi053 ceph-mon[25205]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:51:50.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:50 smithi053 ceph-mon[25205]: Standby manager daemon x restarted 2024-05-15T15:51:50.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:50 smithi053 ceph-mon[25205]: Standby manager daemon x started 2024-05-15T15:51:50.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:50 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:51:50.801 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:50 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:51:50.802 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: [15/May/2024:15:51:50] ENGINE Bus STARTING 2024-05-15T15:51:50.802 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: CherryPy Checker: 2024-05-15T15:51:50.802 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: The Application mounted at '' has an empty config. 2024-05-15T15:51:50.802 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:51:50.803 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: [15/May/2024:15:51:50] ENGINE Serving on http://:::9283 2024-05-15T15:51:50.803 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:51:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: [15/May/2024:15:51:50] ENGINE Bus STARTED 2024-05-15T15:51:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:50 smithi043 ceph-mon[24430]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:51:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:50 smithi043 ceph-mon[24430]: Standby manager daemon x restarted 2024-05-15T15:51:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:50 smithi043 ceph-mon[24430]: Standby manager daemon x started 2024-05-15T15:51:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:50 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:51:50.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:50 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:51:50.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:50 smithi043 ceph-mon[29127]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:51:50.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:50 smithi043 ceph-mon[29127]: Standby manager daemon x restarted 2024-05-15T15:51:50.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:50 smithi043 ceph-mon[29127]: Standby manager daemon x started 2024-05-15T15:51:50.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:50 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:51:50.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:50 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:51:51.729 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:51 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:51:51.729 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:51 smithi053 ceph-mon[25205]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:51:51.729 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:51 smithi053 ceph-mon[25205]: mgrmap e42: y(active, since 28s), standbys: x 2024-05-15T15:51:51.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:51 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:51:51.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:51 smithi043 ceph-mon[24430]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:51:51.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:51 smithi043 ceph-mon[24430]: mgrmap e42: y(active, since 28s), standbys: x 2024-05-15T15:51:51.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:51 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:51:51.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:51 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/2888433909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:51:51.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:51 smithi043 ceph-mon[29127]: mgrmap e42: y(active, since 28s), standbys: x 2024-05-15T15:51:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[24430]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:51:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[29127]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:51:52.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:52 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:52 smithi053 ceph-mon[25205]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-05-15T15:51:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:52 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:52 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:52 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:52 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:51:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:52 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:52 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:51:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:52 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:51:53.762 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:53 smithi043 ceph-mon[24430]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 0 op/s 2024-05-15T15:51:53.762 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:53 smithi043 ceph-mon[24430]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:53.762 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:53 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:51:53.762 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:53 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:51:53.763 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:53 smithi043 ceph-mon[29127]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 0 op/s 2024-05-15T15:51:53.763 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:53 smithi043 ceph-mon[29127]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:53.763 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:53 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:51:53.763 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:53 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:51:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:53 smithi053 ceph-mon[25205]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 0 op/s 2024-05-15T15:51:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:53 smithi053 ceph-mon[25205]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:53 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:51:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:53 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:51:53.995 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:51:54.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:51:53.925+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:54.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:51:54.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:54.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:54.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:54.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:54.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:54.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:54.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:54.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:54.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:51:54.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:51:54.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:51:54.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:54.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:51:54.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:54.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:54.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:54.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:54.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:54.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:54.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:54.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:54.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:54.695 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:51:54.695 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (13m) 26s ago 21m 24.0M - 0.23.0 ba2b418f427c dd5a5b0f3fe5 2024-05-15T15:51:54.695 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (13m) 7s ago 20m 52.3M - 8.3.5 dad864ee21e9 3fece04315b1 2024-05-15T15:51:54.695 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 error 26s ago 20m 60.0M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:51:54.695 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (12s) 7s ago 25m 66.3M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:51:54.695 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (10m) 26s ago 27m 478M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:51:54.695 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (27m) 26s ago 27m 73.7M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:51:54.695 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (25m) 7s ago 25m 60.3M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (26m) 26s ago 26m 56.4M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (13m) 26s ago 21m 15.8M - 1.3.1 1dbe0e931976 083a05fd3833 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (13m) 7s ago 21m 21.9M - 1.3.1 1dbe0e931976 2c6976b8c648 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (25m) 26s ago 25m 53.2M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (24m) 26s ago 24m 56.8M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (24m) 26s ago 24m 49.8M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (24m) 26s ago 24m 52.4M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (23m) 7s ago 23m 54.1M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (23m) 7s ago 23m 51.4M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (22m) 7s ago 22m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (22m) 7s ago 22m 55.5M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (17s) 7s ago 21m 55.7M - 2.33.4 514e6a882f6e f007cf8f16d2 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (20m) 26s ago 20m 75.8M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:51:54.696 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (20m) 7s ago 20m 75.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:51:54.832 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:54.832 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.832 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.832 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-15T15:51:54.832 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-15T15:51:54.832 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-15T15:51:54.832 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-15T15:51:54.832 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.832 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.833 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.834 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:51:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:54.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:51:54.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:51:55.119 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:51:55.120 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:51:55.120 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:51:55.120 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-15T15:51:55.120 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:51:55.120 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:51:55.120 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:51:55.465 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:51:55.465 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T15:51:55.465 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:51:55.465 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-15T15:51:55.465 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [ 2024-05-15T15:51:55.465 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr" 2024-05-15T15:51:55.465 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T15:51:55.465 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "2/2 daemons upgraded", 2024-05-15T15:51:55.465 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading node-exporter daemons", 2024-05-15T15:51:55.465 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:51:55.466 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:51:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:51:55.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:55.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:51:55.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:55.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:55.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:55.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:55.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:55.840 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:55.840 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:55.840 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:55.840 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:55.840 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:51:55.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:55.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:55.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:55.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:51:55.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:55.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:55.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:55.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:55.844 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:55.844 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:55.844 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:55.844 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all mgr 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all crash 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all mds 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all nfs 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: from='client.15390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: from='client.25303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:55.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3489866504' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all mgr 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all crash 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all mds 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T15:51:55.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all nfs 2024-05-15T15:51:55.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: from='client.15390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:55.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:51:55.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:55.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: from='client.25303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:55.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:55 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3489866504' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:55.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:51:55.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:51:55.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:51:55.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: Upgrade: Setting container_image for all mgr 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: Upgrade: Setting container_image for all crash 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: Upgrade: Setting container_image for all mds 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: Upgrade: Setting container_image for all nfs 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: from='client.15390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:55.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: from='client.25303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:55.886 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:55 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3489866504' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:51:56.553 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:51:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:51:56] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.33.4" 2024-05-15T15:51:56.553 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:56 smithi043 systemd[1]: Stopping Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:51:56.836 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:56 smithi043 podman[73746]: 2024-05-15 15:51:56.633063416 +0000 UTC m=+0.128885019 container died 083a05fd38330bc76eb2de142de1d8f8638959775f07ede3994c9bf50a34f17d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:51:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:56 smithi043 ceph-mon[24430]: from='client.15402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:56 smithi043 ceph-mon[24430]: from='client.25321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:56 smithi043 ceph-mon[24430]: Upgrade: Updating node-exporter.a (1/2) 2024-05-15T15:51:56.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:56 smithi043 ceph-mon[24430]: Deploying daemon node-exporter.a on smithi043 2024-05-15T15:51:56.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:56 smithi043 ceph-mon[29127]: from='client.15402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:56.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:56 smithi043 ceph-mon[29127]: from='client.25321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:56.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:56 smithi043 ceph-mon[29127]: Upgrade: Updating node-exporter.a (1/2) 2024-05-15T15:51:56.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:56 smithi043 ceph-mon[29127]: Deploying daemon node-exporter.a on smithi043 2024-05-15T15:51:56.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:56 smithi053 ceph-mon[25205]: from='client.15402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:56.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:56 smithi053 ceph-mon[25205]: from='client.25321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:51:56.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:56 smithi053 ceph-mon[25205]: Upgrade: Updating node-exporter.a (1/2) 2024-05-15T15:51:56.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:56 smithi053 ceph-mon[25205]: Deploying daemon node-exporter.a on smithi043 2024-05-15T15:51:57.163 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:56 smithi043 podman[73746]: 2024-05-15 15:51:56.963617492 +0000 UTC m=+0.459439095 container cleanup 083a05fd38330bc76eb2de142de1d8f8638959775f07ede3994c9bf50a34f17d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:51:57.163 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:56 smithi043 bash[73746]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a 2024-05-15T15:51:57.553 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:57 smithi043 podman[73760]: 2024-05-15 15:51:57.161657805 +0000 UTC m=+0.532066214 container remove 083a05fd38330bc76eb2de142de1d8f8638959775f07ede3994c9bf50a34f17d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:51:57.553 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:57 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-15T15:51:57.553 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:57 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-15T15:51:57.553 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:57 smithi043 systemd[1]: Stopped Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:51:57.553 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:57 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.a.service: Consumed 5.523s CPU time. 2024-05-15T15:51:57.803 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:57 smithi043 ceph-mon[24430]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:57.803 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:57 smithi043 ceph-mon[29127]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:57.803 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:57 smithi043 systemd[1]: Starting Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:51:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:57 smithi053 ceph-mon[25205]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:58.055 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:57 smithi043 bash[73887]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-05-15T15:51:58.836 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:58 smithi043 bash[73887]: Getting image source signatures 2024-05-15T15:51:58.836 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:58 smithi043 bash[73887]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-05-15T15:51:58.836 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:58 smithi043 bash[73887]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-05-15T15:51:58.836 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:51:58 smithi043 bash[73887]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-05-15T15:51:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:59 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:51:59 smithi043 ceph-mon[24430]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:59 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:51:59 smithi043 ceph-mon[29127]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:51:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:59 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:51:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:51:59 smithi053 ceph-mon[25205]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:01.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:01 smithi043 ceph-mon[24430]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:01 smithi043 ceph-mon[29127]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:01 smithi053 ceph-mon[25205]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:03.086 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:02 smithi043 bash[73887]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-05-15T15:52:03.086 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:02 smithi043 bash[73887]: Writing manifest to image destination 2024-05-15T15:52:03.413 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 podman[73887]: 2024-05-15 15:52:03.077458629 +0000 UTC m=+5.375984372 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-05-15T15:52:03.413 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 podman[73887]: 2024-05-15 15:52:03.159021575 +0000 UTC m=+5.457547309 container create c92a2c2e9bf7b56b79f74b04fec996d6490f4950405e0361eb7e4d0d56226417 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:03.413 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 podman[73887]: 2024-05-15 15:52:03.411479754 +0000 UTC m=+5.710005497 container init c92a2c2e9bf7b56b79f74b04fec996d6490f4950405e0361eb7e4d0d56226417 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:03.703 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-mon[24430]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:52:03.703 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:03 smithi043 ceph-mon[29127]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 podman[73887]: 2024-05-15 15:52:03.415798128 +0000 UTC m=+5.714323879 container start c92a2c2e9bf7b56b79f74b04fec996d6490f4950405e0361eb7e4d0d56226417 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.420Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.420Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.421Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.421Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.421Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=arp 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=bcache 2024-05-15T15:52:03.704 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=bonding 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=btrfs 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=conntrack 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=cpu 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=diskstats 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=dmi 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=edac 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=entropy 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=filefd 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=filesystem 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=hwmon 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=infiniband 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=ipvs 2024-05-15T15:52:03.705 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=loadavg 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=mdadm 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=meminfo 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=netclass 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=netdev 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=netstat 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=nfs 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=nfsd 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=nvme 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=os 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=pressure 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=rapl 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=schedstat 2024-05-15T15:52:03.706 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=selinux 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=sockstat 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=softnet 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=stat 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=tapestats 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=textfile 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=time 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=uname 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=vmstat 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=xfs 2024-05-15T15:52:03.707 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=node_exporter.go:117 level=info collector=zfs 2024-05-15T15:52:03.708 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-05-15T15:52:03.708 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a[73936]: ts=2024-05-15T15:52:03.422Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-05-15T15:52:03.708 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 bash[73887]: c92a2c2e9bf7b56b79f74b04fec996d6490f4950405e0361eb7e4d0d56226417 2024-05-15T15:52:03.708 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 15:52:03 smithi043 systemd[1]: Started Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:52:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:03 smithi053 ceph-mon[25205]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:52:04.862 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:04 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:04.862 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:04 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:05.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:04 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:05.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:04 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:05.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:04 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:05.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:04 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:05 smithi053 ceph-mon[25205]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:06.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:05 smithi043 ceph-mon[24430]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:06.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:05 smithi043 ceph-mon[29127]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:06.226 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:05 smithi053 systemd[1]: Stopping Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:52:06.483 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:06 smithi053 podman[61583]: 2024-05-15 15:52:06.225270891 +0000 UTC m=+0.110697688 container died 2c6976b8c648e4534859d70e55ba5e605030af13a1e34bd6177d65e6a1e8523c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:52:06.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:52:06] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.33.4" 2024-05-15T15:52:06.775 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:06 smithi053 ceph-mon[25205]: Upgrade: Updating node-exporter.b (2/2) 2024-05-15T15:52:06.775 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:06 smithi053 ceph-mon[25205]: Deploying daemon node-exporter.b on smithi053 2024-05-15T15:52:06.776 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:06 smithi053 podman[61583]: 2024-05-15 15:52:06.564274173 +0000 UTC m=+0.449700969 container cleanup 2c6976b8c648e4534859d70e55ba5e605030af13a1e34bd6177d65e6a1e8523c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:52:06.776 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:06 smithi053 bash[61583]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b 2024-05-15T15:52:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:06 smithi043 ceph-mon[29127]: Upgrade: Updating node-exporter.b (2/2) 2024-05-15T15:52:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:06 smithi043 ceph-mon[29127]: Deploying daemon node-exporter.b on smithi053 2024-05-15T15:52:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:06 smithi043 ceph-mon[24430]: Upgrade: Updating node-exporter.b (2/2) 2024-05-15T15:52:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:06 smithi043 ceph-mon[24430]: Deploying daemon node-exporter.b on smithi053 2024-05-15T15:52:07.130 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:06 smithi053 podman[61595]: 2024-05-15 15:52:06.774040895 +0000 UTC m=+0.548336379 container remove 2c6976b8c648e4534859d70e55ba5e605030af13a1e34bd6177d65e6a1e8523c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:52:07.130 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:06 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-15T15:52:07.130 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:07 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-15T15:52:07.131 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:07 smithi053 systemd[1]: Stopped Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:52:07.131 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:07 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.b.service: Consumed 5.631s CPU time. 2024-05-15T15:52:07.601 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:07 smithi053 systemd[1]: Starting Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:52:07.601 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:07 smithi053 bash[61724]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-05-15T15:52:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:07 smithi053 ceph-mon[25205]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:08.065 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:07 smithi043 ceph-mon[24430]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:08.065 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:07 smithi043 ceph-mon[29127]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:08.380 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:08 smithi053 bash[61724]: Getting image source signatures 2024-05-15T15:52:08.380 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:08 smithi053 bash[61724]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-05-15T15:52:08.380 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:08 smithi053 bash[61724]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-05-15T15:52:08.381 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:08 smithi053 bash[61724]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-05-15T15:52:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:09 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:09 smithi053 ceph-mon[25205]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:09 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:09 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:09 smithi043 ceph-mon[24430]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:09 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:09 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:09 smithi043 ceph-mon[29127]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:09 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:11.342 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:10 smithi053 bash[61724]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-05-15T15:52:11.342 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 bash[61724]: Writing manifest to image destination 2024-05-15T15:52:11.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 podman[61724]: 2024-05-15 15:52:11.319861126 +0000 UTC m=+4.009973484 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-05-15T15:52:11.607 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 podman[61724]: 2024-05-15 15:52:11.426429793 +0000 UTC m=+4.116542206 container create 1fb31435f1519b91c0711eb92b1f9a7a0b90266b8453855b11a9e57dd7a9f0a8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:52:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-mon[25205]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:11.880 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 podman[61724]: 2024-05-15 15:52:11.645328672 +0000 UTC m=+4.335441031 container init 1fb31435f1519b91c0711eb92b1f9a7a0b90266b8453855b11a9e57dd7a9f0a8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:52:11.880 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 podman[61724]: 2024-05-15 15:52:11.648977993 +0000 UTC m=+4.339090366 container start 1fb31435f1519b91c0711eb92b1f9a7a0b90266b8453855b11a9e57dd7a9f0a8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T15:52:11.880 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.653Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-05-15T15:52:11.880 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.653Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.654Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.654Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.654Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=arp 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=bcache 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=bonding 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=btrfs 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=conntrack 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=cpu 2024-05-15T15:52:11.881 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-05-15T15:52:11.882 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=diskstats 2024-05-15T15:52:11.882 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=dmi 2024-05-15T15:52:11.882 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=edac 2024-05-15T15:52:11.882 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=entropy 2024-05-15T15:52:11.882 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-05-15T15:52:11.882 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=filefd 2024-05-15T15:52:11.882 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=filesystem 2024-05-15T15:52:11.882 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=hwmon 2024-05-15T15:52:11.882 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=infiniband 2024-05-15T15:52:11.882 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=ipvs 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=loadavg 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=mdadm 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=meminfo 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=netclass 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=netdev 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=netstat 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=nfs 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=nfsd 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=nvme 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=os 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=pressure 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=rapl 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=schedstat 2024-05-15T15:52:11.883 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=selinux 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=sockstat 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=softnet 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=stat 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=tapestats 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=textfile 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=time 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=uname 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=vmstat 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=xfs 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.655Z caller=node_exporter.go:117 level=info collector=zfs 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.656Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b[61774]: ts=2024-05-15T15:52:11.656Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-05-15T15:52:11.884 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 bash[61724]: 1fb31435f1519b91c0711eb92b1f9a7a0b90266b8453855b11a9e57dd7a9f0a8 2024-05-15T15:52:11.885 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 15:52:11 smithi053 systemd[1]: Started Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:52:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:11 smithi043 ceph-mon[24430]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:11 smithi043 ceph-mon[29127]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:13.009 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:12 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:13.009 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:12 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:13.009 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:12 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:13.009 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:12 smithi053 ceph-mon[25205]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:12 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:12 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:12 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:12 smithi043 ceph-mon[24430]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:13.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:12 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:13.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:12 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:13.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:12 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:12 smithi043 ceph-mon[29127]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:15 smithi053 ceph-mon[25205]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:15.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:15 smithi043 ceph-mon[24430]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:15.403 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:15 smithi043 ceph-mon[29127]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:16.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:52:16] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.33.4" 2024-05-15T15:52:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:17 smithi053 ceph-mon[25205]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:17 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:17 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:17 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:17 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:17 smithi043 ceph-mon[24430]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:17 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:17 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:17 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:17 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:17 smithi043 ceph-mon[29127]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:17 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:17 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:17 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:17 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:18 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:18 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:18 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.594 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:18 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.594 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:18 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.594 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:18 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.594 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:18 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.594 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:18 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.858 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:18 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.859 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:18 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.859 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:18 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:18.859 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:18 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:19 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:19 smithi043 ceph-mon[24430]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:19 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:19.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:19 smithi043 ceph-mon[29127]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:19 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:19 smithi053 ceph-mon[25205]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[29127]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[24430]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:52:21.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:52:21.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:21 smithi053 ceph-mon[25205]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:21 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:21 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:21 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:21 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:21 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:21.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:21 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:52:21.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:21 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:21.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:21 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:52:21.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:21 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:22.763 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:22 smithi043 ceph-mon[24430]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 937 B/s rd, 0 op/s 2024-05-15T15:52:22.764 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:22 smithi043 ceph-mon[24430]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:22.764 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:22 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:52:22.764 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:22 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:52:22.764 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:22 smithi043 ceph-mon[29127]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 937 B/s rd, 0 op/s 2024-05-15T15:52:22.764 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:22 smithi043 ceph-mon[29127]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:22.764 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:22 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:52:22.764 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:22 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:52:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:22 smithi053 ceph-mon[25205]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 937 B/s rd, 0 op/s 2024-05-15T15:52:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:22 smithi053 ceph-mon[25205]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:22 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:52:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:22 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:52:22.904+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:23.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:52:23.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:23.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:23.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:23.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:23.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:23.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:23.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:23.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:52:23.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:52:23.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:23.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:23.812 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:23 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:23.812 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:23 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.812 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:23 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.812 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:23 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.812 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:23 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.813 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:23 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.813 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:23 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.813 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:23 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.813 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:23 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.813 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:23 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:23.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:23 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:24.832 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:24 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:24.832 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:24.832 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:24.832 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:24.832 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:24.833 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:52:24.834 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:24.835 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:24 smithi053 ceph-mon[25205]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:24.836 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:24 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:52:24.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:52:24.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:52:24.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:52:24.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:52:24.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:24.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:24.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:24.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:24.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:24.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:24.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:24.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:24.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:24.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:52:24.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:24.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:24.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:24.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:24 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:52:24.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:52:24.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:24.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:24.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:24.846 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:24.846 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:24.846 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:24.846 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.846 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.846 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:24 smithi043 ceph-mon[24430]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:24.846 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:24.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:24.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:24.847 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:24.847 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:24.847 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:24.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:24 smithi043 ceph-mon[29127]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:24.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:25.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:25 smithi043 ceph-mon[24430]: Upgrade: Updating prometheus.a 2024-05-15T15:52:25.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:25 smithi043 ceph-mon[29127]: Upgrade: Updating prometheus.a 2024-05-15T15:52:25.843 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:52:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:25 smithi053 ceph-mon[25205]: Upgrade: Updating prometheus.a 2024-05-15T15:52:26.522 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (13m) 10s ago 21m 24.4M - 0.23.0 ba2b418f427c dd5a5b0f3fe5 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (13m) 10s ago 21m 52.4M - 8.3.5 dad864ee21e9 3fece04315b1 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 10s ago 20m 60.2M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (43s) 10s ago 26m 453M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (10m) 10s ago 27m 502M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (27m) 10s ago 27m 77.8M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (26m) 10s ago 26m 61.5M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (26m) 10s ago 26m 59.8M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (23s) 10s ago 22m 3594k - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (15s) 10s ago 22m 3393k - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (25m) 10s ago 25m 53.8M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (25m) 10s ago 25m 58.0M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:52:26.523 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (25m) 10s ago 25m 50.4M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:52:26.524 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (24m) 10s ago 24m 53.7M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:52:26.524 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (24m) 10s ago 24m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:52:26.524 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (23m) 10s ago 23m 51.3M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:52:26.524 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (23m) 10s ago 23m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:52:26.524 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (23m) 10s ago 23m 55.5M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:52:26.524 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (49s) 10s ago 22m 63.3M - 2.33.4 514e6a882f6e f007cf8f16d2 2024-05-15T15:52:26.524 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (20m) 10s ago 20m 76.0M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:52:26.524 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (20m) 10s ago 20m 75.1M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:52:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:26 smithi043 ceph-mon[24430]: Deploying daemon prometheus.a on smithi053 2024-05-15T15:52:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:26 smithi043 ceph-mon[24430]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:26 smithi043 ceph-mon[29127]: Deploying daemon prometheus.a on smithi053 2024-05-15T15:52:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:26 smithi043 ceph-mon[29127]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:26.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:52:26] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-05-15T15:52:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:26 smithi053 ceph-mon[25205]: Deploying daemon prometheus.a on smithi053 2024-05-15T15:52:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:26 smithi053 ceph-mon[25205]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:52:26.935 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:52:26.936 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:52:26.936 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:52:26.936 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:52:26.936 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-15T15:52:26.936 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:52:26.936 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:52:26.936 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:52:27.280 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:52:27.281 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T15:52:27.281 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:52:27.281 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-15T15:52:27.281 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [ 2024-05-15T15:52:27.281 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr" 2024-05-15T15:52:27.281 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T15:52:27.281 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "2/2 daemons upgraded", 2024-05-15T15:52:27.281 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading prometheus daemons", 2024-05-15T15:52:27.281 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:52:27.281 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:52:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:27 smithi053 ceph-mon[25205]: from='client.15420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:27 smithi053 ceph-mon[25205]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:27.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:27 smithi053 ceph-mon[25205]: from='client.25339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:27.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:27 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3372145432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:28.077 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:27 smithi043 ceph-mon[24430]: from='client.15420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:28.077 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:27 smithi043 ceph-mon[24430]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:28.077 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:27 smithi043 ceph-mon[24430]: from='client.25339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:28.077 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:27 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3372145432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:28.077 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:27 smithi043 ceph-mon[29127]: from='client.15420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:28.077 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:27 smithi043 ceph-mon[29127]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:28.077 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:27 smithi043 ceph-mon[29127]: from='client.25339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:28.078 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:27 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3372145432' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:28 smithi053 ceph-mon[25205]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:28 smithi053 ceph-mon[25205]: from='client.15444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:29.028 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:28 smithi043 ceph-mon[24430]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:29.029 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:28 smithi043 ceph-mon[24430]: from='client.15444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:29.029 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:28 smithi043 ceph-mon[29127]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:29.029 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:28 smithi043 ceph-mon[29127]: from='client.15444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:29 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:30.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:29 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:30.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:29 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:30.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:30 smithi053 ceph-mon[25205]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s 2024-05-15T15:52:31.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:30 smithi043 ceph-mon[24430]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s 2024-05-15T15:52:31.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:30 smithi043 ceph-mon[29127]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s 2024-05-15T15:52:32.854 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:32 smithi053 ceph-mon[25205]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:52:33.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:32 smithi043 ceph-mon[24430]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:52:33.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:32 smithi043 ceph-mon[29127]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:52:34.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:34 smithi053 ceph-mon[25205]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:35.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:34 smithi043 ceph-mon[24430]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:35.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:34 smithi043 ceph-mon[29127]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:35.860 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 systemd[1]: Stopping Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:52:36.130 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.857Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[59709]: ts=2024-05-15T15:52:35.858Z caller=main.go:1066 level=info msg="See you next time!" 2024-05-15T15:52:36.131 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:35 smithi053 podman[63257]: 2024-05-15 15:52:35.8681243 +0000 UTC m=+0.133418516 container died f007cf8f16d2fe21822f8900b9749378a6807286ef718317f2e74ba8e27b4b68 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:36.601 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:36 smithi053 podman[63257]: 2024-05-15 15:52:36.182144144 +0000 UTC m=+0.447438357 container cleanup f007cf8f16d2fe21822f8900b9749378a6807286ef718317f2e74ba8e27b4b68 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:36.601 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:36 smithi053 bash[63257]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a 2024-05-15T15:52:36.601 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:36 smithi053 podman[63268]: 2024-05-15 15:52:36.391567726 +0000 UTC m=+0.520601605 container remove f007cf8f16d2fe21822f8900b9749378a6807286ef718317f2e74ba8e27b4b68 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:36 smithi053 ceph-mon[25205]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:36.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:36 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-15T15:52:36.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:36 smithi053 systemd[1]: Stopped Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:52:36.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:36 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Consumed 1.381s CPU time. 2024-05-15T15:52:37.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:36 smithi043 ceph-mon[24430]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:37.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:36 smithi043 ceph-mon[29127]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:37.380 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:36 smithi053 systemd[1]: Starting Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:52:37.380 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:37 smithi053 podman[63400]: 2024-05-15 15:52:37.010018699 +0000 UTC m=+0.019220740 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-05-15T15:52:38.380 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 podman[63400]: 2024-05-15 15:52:38.05559024 +0000 UTC m=+1.064792263 container create c420668ab64015623c56a7ca304e621357a7d8821949c7352aae4a1ffbc392b6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:38 smithi053 ceph-mon[25205]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:38.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 podman[63400]: 2024-05-15 15:52:38.574840202 +0000 UTC m=+1.584042222 container init c420668ab64015623c56a7ca304e621357a7d8821949c7352aae4a1ffbc392b6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:38.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 podman[63400]: 2024-05-15 15:52:38.578121809 +0000 UTC m=+1.587323838 container start c420668ab64015623c56a7ca304e621357a7d8821949c7352aae4a1ffbc392b6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:38.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.607Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-05-15T15:52:38.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.607Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-05-15T15:52:38.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.607Z caller=main.go:570 level=info host_details="(Linux 5.14.0-446.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri May 3 16:10:18 UTC 2024 x86_64 smithi053 (none))" 2024-05-15T15:52:38.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.607Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-05-15T15:52:38.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.607Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-05-15T15:52:38.881 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.609Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.609Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.610Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.610Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.612Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.612Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=4.468µs 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.612Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.620Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.634Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.645Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.648Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.651Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.652Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-05-15T15:52:38.882 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.652Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-05-15T15:52:38.883 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.652Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=70.687µs wal_replay_duration=39.90395ms wbl_replay_duration=113ns total_replay_duration=40.095645ms 2024-05-15T15:52:38.883 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.656Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-05-15T15:52:38.883 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.656Z caller=main.go:1029 level=info msg="TSDB started" 2024-05-15T15:52:38.883 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.656Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-05-15T15:52:38.883 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 bash[63400]: c420668ab64015623c56a7ca304e621357a7d8821949c7352aae4a1ffbc392b6 2024-05-15T15:52:38.883 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 systemd[1]: Started Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:52:38.883 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.681Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.664383ms db_storage=871ns remote_storage=1.134µs web_handler=389ns query_engine=579ns scrape=435.921µs scrape_sd=96.895µs notify=22.883µs notify_sd=19.602µs rules=23.762239ms tracing=8.5µs 2024-05-15T15:52:38.883 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.681Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-05-15T15:52:38.883 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 15:52:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T15:52:38.681Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-05-15T15:52:39.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:38 smithi043 ceph-mon[24430]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:39.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:38 smithi043 ceph-mon[29127]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:39 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:39 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:39 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:39 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:39.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:39 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:39 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:39 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:39 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:39 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:39 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:39 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:39 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:39 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:39 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:39 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:40 smithi053 ceph-mon[25205]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:41.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:40 smithi043 ceph-mon[24430]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:41.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:40 smithi043 ceph-mon[29127]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:42 smithi053 ceph-mon[25205]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:43.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:42 smithi043 ceph-mon[24430]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:43.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:42 smithi043 ceph-mon[29127]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:44.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:44 smithi043 ceph-mon[29127]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:44.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:44 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:44.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:44 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:44.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:44 smithi043 ceph-mon[24430]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:44.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:44 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:44.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:44 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:44 smithi053 ceph-mon[25205]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:44 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:44 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:45.315 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:45 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:45.315 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:45 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:45.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:45 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:45.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:45 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:45.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:45 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:45.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:45 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:46 smithi053 ceph-mon[25205]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:46 smithi043 ceph-mon[24430]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:46.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:46 smithi043 ceph-mon[29127]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:52:46.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:52:46] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.43.0" 2024-05-15T15:52:48.771 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[24430]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:48.771 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:48.771 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:48.771 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:48.771 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[29127]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:52:48.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:48 smithi053 ceph-mon[25205]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:52:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:48 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:48 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:48 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:48.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:48 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:52:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:48 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:52:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:48 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:52:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:48 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:52:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:49 smithi043 ceph-mon[24430]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-05-15T15:52:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:49 smithi043 ceph-mon[24430]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:49 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:52:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:49 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:52:49.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:49 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:49 smithi043 ceph-mon[29127]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-05-15T15:52:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:49 smithi043 ceph-mon[29127]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:49 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:52:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:49 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:52:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:49 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:52:49.665+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:52:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:52:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:52:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:52:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:49.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:49 smithi053 ceph-mon[25205]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-05-15T15:52:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:49 smithi053 ceph-mon[25205]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:49.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:49 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:52:49.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:49 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:52:49.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:49 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:50.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:50.566 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:50.566 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:50.566 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:50.566 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:50.569 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.569 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.569 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:50.569 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:50.569 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:50.569 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:50.569 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:52:50.569 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:52:50.570 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:50.571 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:50.572 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:50.573 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:52:50.574 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:50.575 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:50.576 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.577 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.578 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.579 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:50 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:52:50.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:52:50.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:52:50.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:52:50.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.886 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:50.886 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:50 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:51.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:51 smithi043 ceph-mon[24430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:52:51.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:51 smithi043 ceph-mon[24430]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-05-15T15:52:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:51 smithi043 ceph-mon[29127]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:52:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:51 smithi043 ceph-mon[29127]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-05-15T15:52:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:51 smithi053 ceph-mon[25205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-15T15:52:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:51 smithi053 ceph-mon[25205]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-05-15T15:52:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:52 smithi043 ceph-mon[24430]: Upgrade: Updating alertmanager.a 2024-05-15T15:52:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:52 smithi043 ceph-mon[24430]: Deploying daemon alertmanager.a on smithi043 2024-05-15T15:52:52.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:52 smithi043 ceph-mon[29127]: Upgrade: Updating alertmanager.a 2024-05-15T15:52:52.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:52 smithi043 ceph-mon[29127]: Deploying daemon alertmanager.a on smithi043 2024-05-15T15:52:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:52 smithi053 ceph-mon[25205]: Upgrade: Updating alertmanager.a 2024-05-15T15:52:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:52 smithi053 ceph-mon[25205]: Deploying daemon alertmanager.a on smithi043 2024-05-15T15:52:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:53 smithi043 ceph-mon[24430]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:53.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:53 smithi043 ceph-mon[29127]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:53 smithi053 ceph-mon[25205]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:54.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:52:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:55 smithi043 ceph-mon[24430]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-05-15T15:52:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:55 smithi043 ceph-mon[29127]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-05-15T15:52:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:55 smithi053 ceph-mon[25205]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-05-15T15:52:56.593 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:52:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:52:56] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.43.0" 2024-05-15T15:52:57.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:57 smithi043 ceph-mon[24430]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:57.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:57 smithi043 ceph-mon[29127]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:57.669 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:52:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:57 smithi053 ceph-mon[25205]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:52:58.371 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (14m) 42s ago 22m 24.4M - 0.23.0 ba2b418f427c dd5a5b0f3fe5 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (14m) 15s ago 21m 52.4M - 8.3.5 dad864ee21e9 3fece04315b1 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 42s ago 21m 60.2M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (75s) 15s ago 26m 454M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (11m) 42s ago 28m 502M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (28m) 42s ago 28m 77.8M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (26m) 15s ago 26m 62.7M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (27m) 42s ago 27m 59.8M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (55s) 42s ago 22m 3594k - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (47s) 15s ago 22m 9252k - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:52:58.372 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (26m) 42s ago 26m 53.8M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:52:58.373 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (25m) 42s ago 25m 58.0M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:52:58.373 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (25m) 42s ago 25m 50.4M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:52:58.373 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (25m) 42s ago 25m 53.7M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:52:58.373 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (24m) 15s ago 24m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:52:58.373 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (24m) 15s ago 24m 51.3M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:52:58.373 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (24m) 15s ago 23m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:52:58.373 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (23m) 15s ago 23m 55.5M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:52:58.373 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (21s) 15s ago 22m 47.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:52:58.373 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (21m) 42s ago 21m 76.0M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:52:58.373 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (21m) 15s ago 21m 75.2M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:52:58.633 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:58 smithi043 ceph-mon[24430]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:58.633 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:58 smithi043 ceph-mon[29127]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:58.787 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:52:58.787 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:52:58.787 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:52:58.787 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:52:58.788 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:52:58.789 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:52:58.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:58 smithi053 ceph-mon[25205]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:59.178 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:52:59.178 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T15:52:59.178 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:52:59.178 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-15T15:52:59.178 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [ 2024-05-15T15:52:59.178 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr" 2024-05-15T15:52:59.178 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T15:52:59.179 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "2/2 daemons upgraded", 2024-05-15T15:52:59.179 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading alertmanager daemons", 2024-05-15T15:52:59.179 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:52:59.179 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:52:59.318 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:52:59 smithi043 systemd[1]: Stopping Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:52:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:59 smithi043 ceph-mon[24430]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:52:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:59 smithi043 ceph-mon[24430]: from='client.15456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:59 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:59 smithi043 ceph-mon[24430]: from='client.25369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:52:59 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2388566693' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:59.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:59 smithi043 ceph-mon[29127]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:52:59.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:59 smithi043 ceph-mon[29127]: from='client.15456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:59.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:59 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:59.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:59 smithi043 ceph-mon[29127]: from='client.25369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:59.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:52:59 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2388566693' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:59.587 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:52:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[58346]: level=info ts=2024-05-15T15:52:59.316Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-05-15T15:52:59.587 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:52:59 smithi043 podman[76006]: 2024-05-15 15:52:59.368081567 +0000 UTC m=+0.144146823 container died dd5a5b0f3fe5f39f7907eec560ffd624568ac52824e2279aa483414fa4913c4d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:59 smithi053 ceph-mon[25205]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:52:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:59 smithi053 ceph-mon[25205]: from='client.15456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:59 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:52:59.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:59 smithi053 ceph-mon[25205]: from='client.25369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:52:59.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:52:59 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2388566693' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:52:59.986 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:52:59 smithi043 podman[76006]: 2024-05-15 15:52:59.733592622 +0000 UTC m=+0.509657878 container cleanup dd5a5b0f3fe5f39f7907eec560ffd624568ac52824e2279aa483414fa4913c4d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:52:59.986 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:52:59 smithi043 bash[76006]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a 2024-05-15T15:53:00.241 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:52:59 smithi043 podman[76019]: 2024-05-15 15:52:59.984957918 +0000 UTC m=+0.613383425 container remove dd5a5b0f3fe5f39f7907eec560ffd624568ac52824e2279aa483414fa4913c4d (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:53:00.241 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:00 smithi043 podman[76019]: 2024-05-15 15:53:00.077573112 +0000 UTC m=+0.705998617 volume remove 8d4ccbef251e4cfa12982f63fcbbd8b14d0948f15b50135c3267d232be9d5007 2024-05-15T15:53:00.241 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:00 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-15T15:53:00.241 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:00 smithi043 systemd[1]: Stopped Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:53:00.241 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:00 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@alertmanager.a.service: Consumed 1.638s CPU time. 2024-05-15T15:53:00.545 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:00 smithi043 systemd[1]: Starting Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:53:00.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:00 smithi043 ceph-mon[29127]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:00 smithi043 ceph-mon[24430]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:00.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:00 smithi043 podman[76163]: 2024-05-15 15:53:00.678384425 +0000 UTC m=+0.114445788 volume create f00a0c399f53f6b060c21d93bd25e957552360bc5584da4d430b758c8eec57d8 2024-05-15T15:53:00.837 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:00 smithi043 podman[76163]: 2024-05-15 15:53:00.581624646 +0000 UTC m=+0.017686022 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-05-15T15:53:00.837 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:00 smithi043 podman[76163]: 2024-05-15 15:53:00.787218064 +0000 UTC m=+0.223279424 container create bd365acb01c53c23c17d8acde42d75dadd6bc1006b78232554246bcdb10e9326 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:53:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:00 smithi053 ceph-mon[25205]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:01.336 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 podman[76163]: 2024-05-15 15:53:01.073478544 +0000 UTC m=+0.509539914 container init bd365acb01c53c23c17d8acde42d75dadd6bc1006b78232554246bcdb10e9326 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:53:01.336 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 podman[76163]: 2024-05-15 15:53:01.076689243 +0000 UTC m=+0.512750612 container start bd365acb01c53c23c17d8acde42d75dadd6bc1006b78232554246bcdb10e9326 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T15:53:01.336 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:53:01.088Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-05-15T15:53:01.336 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:53:01.088Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-05-15T15:53:01.336 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:53:01.089Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.43 port=9094 2024-05-15T15:53:01.337 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:53:01.091Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-05-15T15:53:01.337 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:53:01.114Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-15T15:53:01.337 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:53:01.114Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-05-15T15:53:01.337 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:53:01.117Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-05-15T15:53:01.337 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:53:01.117Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-05-15T15:53:01.337 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 bash[76163]: bd365acb01c53c23c17d8acde42d75dadd6bc1006b78232554246bcdb10e9326 2024-05-15T15:53:01.337 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:01 smithi043 systemd[1]: Started Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:53:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:01 smithi043 ceph-mon[29127]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:01.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:01 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:01.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:01 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:01.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:01 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:01.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-mon[24430]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:01.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:01.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:01.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:01 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:01 smithi053 ceph-mon[25205]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:01 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:01 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:01 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:03.336 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:53:03.091Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000253077s 2024-05-15T15:53:03.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:03 smithi043 ceph-mon[24430]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:03 smithi043 ceph-mon[29127]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:03 smithi053 ceph-mon[25205]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:05.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:05 smithi043 ceph-mon[24430]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:05.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:05 smithi043 ceph-mon[29127]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:05 smithi053 ceph-mon[25205]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:06.742 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:53:06] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.43.0" 2024-05-15T15:53:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:06 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:06 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:06 smithi043 ceph-mon[24430]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:06 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:06 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:06 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:06 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:06 smithi043 ceph-mon[29127]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:07.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:06 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:06 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:06 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:06 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:06 smithi053 ceph-mon[25205]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:06 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:06 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:08.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:08 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:08.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:08 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:08.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:08 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:08.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:08 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:08.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:08 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:08.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:08 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:09.363 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:09 smithi043 ceph-mon[24430]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:09.363 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:09 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:09.363 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:09 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:09.363 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:09 smithi043 ceph-mon[29127]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:09.364 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:09 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:09.364 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:09 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:09 smithi053 ceph-mon[25205]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:09 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:09 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:11.336 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:53:11 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:53:11.094Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003134205s 2024-05-15T15:53:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[29127]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:11.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[24430]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:11.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:11 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:11 smithi053 ceph-mon[25205]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:11 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:11 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:11 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:11.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:11 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:11.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:11 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:11.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:11 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:11.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:11 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:12.793 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:12 smithi043 ceph-mon[24430]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:53:12.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:12 smithi043 ceph-mon[24430]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:12.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:12 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:12.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:12 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:12.794 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:12 smithi043 ceph-mon[29127]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:53:12.794 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:12 smithi043 ceph-mon[29127]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:12.794 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:12 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:12.794 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:12 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:12.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:12 smithi053 ceph-mon[25205]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:53:12.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:12 smithi053 ceph-mon[25205]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:12.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:12 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:12.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:12 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:13.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:53:12.929+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:13.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:53:13.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:13.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:13.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:13.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:13.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:13.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:13.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:13.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:13.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:13.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:13.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:13.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:13.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:13.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:13.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:13.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:13.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:13.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:13.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:53:13.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:13.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.780 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.781 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:13 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:53:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:13.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:13 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:14 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:53:14.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:14.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:14 smithi043 ceph-mon[24430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:53:14.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:14 smithi043 ceph-mon[24430]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:14.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:14 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:14.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:14.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:14.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:14.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:14.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:53:14.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:14.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:14 smithi043 ceph-mon[29127]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:53:14.845 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:14 smithi043 ceph-mon[29127]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:14 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:14.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:14.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:14.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:14.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:14.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:14.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:14.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:14.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:14.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:14 smithi053 ceph-mon[25205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-15T15:53:14.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:14 smithi053 ceph-mon[25205]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:15 smithi043 ceph-mon[24430]: Upgrade: Updating grafana.a 2024-05-15T15:53:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:15 smithi043 ceph-mon[24430]: Deploying daemon grafana.a on smithi053 2024-05-15T15:53:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:15 smithi043 ceph-mon[29127]: Upgrade: Updating grafana.a 2024-05-15T15:53:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:15 smithi043 ceph-mon[29127]: Deploying daemon grafana.a on smithi053 2024-05-15T15:53:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:15 smithi053 ceph-mon[25205]: Upgrade: Updating grafana.a 2024-05-15T15:53:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:15 smithi053 ceph-mon[25205]: Deploying daemon grafana.a on smithi053 2024-05-15T15:53:16.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:16 smithi043 ceph-mon[24430]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:16.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:16 smithi043 ceph-mon[29127]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:53:16] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.43.0" 2024-05-15T15:53:16.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:16 smithi053 ceph-mon[25205]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:18.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:18 smithi043 ceph-mon[24430]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:18.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:18 smithi043 ceph-mon[29127]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:18 smithi053 ceph-mon[25205]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:19.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:19 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:19.794 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:19 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:19 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:20.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:20 smithi043 ceph-mon[24430]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:20.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:20 smithi043 ceph-mon[29127]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:20 smithi053 ceph-mon[25205]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:22.773 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:22 smithi053 ceph-mon[25205]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:53:22.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:22 smithi043 ceph-mon[24430]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:53:22.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:22 smithi043 ceph-mon[29127]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:53:24.822 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:24 smithi053 ceph-mon[25205]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:24.822 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:24 smithi043 ceph-mon[24430]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:24.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:24 smithi043 ceph-mon[29127]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:24.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:25.082 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:24 smithi053 systemd[1]: Stopping Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:53:25.082 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:25 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[48846]: t=2024-05-15T15:53:25+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-05-15T15:53:25.348 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:25 smithi053 podman[65469]: 2024-05-15 15:53:25.080937249 +0000 UTC m=+0.108540335 container died 3fece04315b10c927655a1d132a99bc43c1663b10c3257d16d07b5963cee873a (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., io.buildah.version=1.24.2, description=Ceph Grafana Container, io.openshift.expose-services=, name=ubi8, version=8.5, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, maintainer=Paul Cuzner ) 2024-05-15T15:53:25.630 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:25 smithi053 podman[65469]: 2024-05-15 15:53:25.428246536 +0000 UTC m=+0.455849608 container cleanup 3fece04315b10c927655a1d132a99bc43c1663b10c3257d16d07b5963cee873a (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, name=ubi8, description=Ceph Grafana Container, release=236.1648460182, build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, io.openshift.tags=base rhel8, vcs-type=git, version=8.5, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc.) 2024-05-15T15:53:25.630 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:25 smithi053 bash[65469]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a 2024-05-15T15:53:26.075 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:25 smithi053 podman[65482]: 2024-05-15 15:53:25.721202205 +0000 UTC m=+0.637096585 container remove 3fece04315b10c927655a1d132a99bc43c1663b10c3257d16d07b5963cee873a (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, distribution-scope=public, io.buildah.version=1.24.2, name=ubi8, vcs-type=git, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., version=8.5, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-05-15T15:53:26.075 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:25 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-15T15:53:26.075 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:25 smithi053 systemd[1]: Stopped Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:53:26.075 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:25 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@grafana.a.service: Consumed 5.707s CPU time. 2024-05-15T15:53:26.380 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 systemd[1]: Starting Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:53:26.380 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 podman[65608]: 2024-05-15 15:53:26.195308132 +0000 UTC m=+0.018849966 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-05-15T15:53:26.380 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 podman[65608]: 2024-05-15 15:53:26.319566772 +0000 UTC m=+0.143108648 container create 16402042b8e562911a3ec6a45b26dbaacd937df0b6cb0f4f925fcd10cb85866d (image=quay.io/ceph/grafana:9.4.12, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-15T15:53:26.690 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:26 smithi053 ceph-mon[25205]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:26.690 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 podman[65608]: 2024-05-15 15:53:26.532786827 +0000 UTC m=+0.356328669 container init 16402042b8e562911a3ec6a45b26dbaacd937df0b6cb0f4f925fcd10cb85866d (image=quay.io/ceph/grafana:9.4.12, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-15T15:53:26.690 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 podman[65608]: 2024-05-15 15:53:26.535589172 +0000 UTC m=+0.359131007 container start 16402042b8e562911a3ec6a45b26dbaacd937df0b6cb0f4f925fcd10cb85866d (image=quay.io/ceph/grafana:9.4.12, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-15T15:53:26.690 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 bash[65608]: 16402042b8e562911a3ec6a45b26dbaacd937df0b6cb0f4f925fcd10cb85866d 2024-05-15T15:53:26.690 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 systemd[1]: Started Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:53:26.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:26 smithi043 ceph-mon[24430]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:26.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:26 smithi043 ceph-mon[29127]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:26.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:53:26] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.43.0" 2024-05-15T15:53:27.026 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688700317Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-05-15T15:53:27.026 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688886457Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-05-15T15:53:27.026 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688897824Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-05-15T15:53:27.026 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688901523Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-05-15T15:53:27.026 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688904434Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688907357Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688910207Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688912955Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688915708Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688918794Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688921605Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688924228Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688927035Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688929878Z level=info msg="Path Home" path=/usr/share/grafana 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688933047Z level=info msg="Path Data" path=/var/lib/grafana 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688936148Z level=info msg="Path Logs" path=/var/log/grafana 2024-05-15T15:53:27.027 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688938705Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-05-15T15:53:27.028 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688941456Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-05-15T15:53:27.028 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=settings t=2024-05-15T15:53:26.688944215Z level=info msg="App mode production" 2024-05-15T15:53:27.028 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=sqlstore t=2024-05-15T15:53:26.688989942Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-05-15T15:53:27.028 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=sqlstore t=2024-05-15T15:53:26.689006816Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-05-15T15:53:27.028 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.70092145Z level=info msg="Starting DB migrations" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.70455031Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.716126706Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.717308714Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.717631153Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.718813129Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.719949774Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.720974767Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.721261169Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.721718922Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.722200934Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.722528109Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.723601418Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-05-15T15:53:27.029 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.723921574Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.724973894Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.726030685Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.727086719Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.727372749Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.728423386Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.729478005Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.729769582Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.730815612Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.73188859Z level=info msg="Executing migration" id="create provenance_type table" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.732329045Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.732787354Z level=info msg="Executing migration" id="create alert_image table" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.733221123Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-05-15T15:53:27.030 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.733655115Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.733931962Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.734352697Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.734753747Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.734839504Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.735123292Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.735622287Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.736104349Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.736382552Z level=info msg="Executing migration" id="create secrets table" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.736785234Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.743135894Z level=info msg="Executing migration" id="add name column into data_keys" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.744271133Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.74456834Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.750999937Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-05-15T15:53:27.031 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.757381247Z level=info msg="Executing migration" id="add column hidden to role table" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.758503763Z level=info msg="Executing migration" id="create query_history table v1" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.75888624Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.759404178Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.759652614Z level=info msg="Executing migration" id="teams permissions migration" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.760127826Z level=info msg="Executing migration" id="dashboard permissions" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.761479579Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.762164729Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.762450714Z level=info msg="Executing migration" id="alerting notification permissions" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.762743514Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.763134305Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.763564343Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-05-15T15:53:27.032 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.764667576Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-05-15T15:53:27.033 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.764952022Z level=info msg="Executing migration" id="create correlation table v1" 2024-05-15T15:53:27.033 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.765380707Z level=info msg="Executing migration" id="add index correlations.uid" 2024-05-15T15:53:27.033 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.765792505Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-05-15T15:53:27.033 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.766219829Z level=info msg="Executing migration" id="add correlation config column" 2024-05-15T15:53:27.033 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.76734478Z level=info msg="Executing migration" id="create entity_events table" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.7677184Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.768119031Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.768201199Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.768443394Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.768528904Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.768783934Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.769200713Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.769593196Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.770046981Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.770468159Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.770835272Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.771270651Z level=info msg="Executing migration" id="Drop public config table" 2024-05-15T15:53:27.034 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.771615799Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.77203005Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.772450129Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.772826056Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.773250168Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.777162033Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.778316412Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.7796024Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.779915114Z level=info msg="Executing migration" id="add share column" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.781142896Z level=info msg="Executing migration" id="create default alerting folders" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.781625315Z level=info msg="Executing migration" id="create file table" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.782050422Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.782500045Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.782979065Z level=info msg="Executing migration" id="create file_meta table" 2024-05-15T15:53:27.035 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.783406226Z level=info msg="Executing migration" id="file table idx: path key" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.783836955Z level=info msg="Executing migration" id="set path collation in file table" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.784118378Z level=info msg="Executing migration" id="managed permissions migration" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.785193534Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.78580829Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.786416656Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.787608144Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.787914961Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.788358943Z level=info msg="Executing migration" id="update group index for alert rules" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.788732443Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.789284665Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.789661588Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.79086563Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.792071207Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-05-15T15:53:27.036 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.79256067Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.806066897Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.806714626Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.807180396Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.811659798Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.812218665Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.812570454Z level=info msg="Executing migration" id="create folder table" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.813141978Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.813590663Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.814042185Z level=info msg="Executing migration" id="Update folder title length" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.814319179Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=migrator t=2024-05-15T15:53:26.81477315Z level=info msg="migrations completed" performed=105 skipped=376 duration=110.261712ms 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=sqlstore t=2024-05-15T15:53:26.815118567Z level=info msg="Created default organization" 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=plugin.finder t=2024-05-15T15:53:26.869369949Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-05-15T15:53:27.037 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=secrets t=2024-05-15T15:53:26.869638833Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-05-15T15:53:27.038 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=query_data t=2024-05-15T15:53:26.872207569Z level=info msg="Query Service initialization" 2024-05-15T15:53:27.038 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:26 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=live.push_http t=2024-05-15T15:53:26.874609138Z level=info msg="Live Push Gateway initialization" 2024-05-15T15:53:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:27 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:27 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:27 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=infra.usagestats.collector t=2024-05-15T15:53:27.552304503Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=provisioning.datasources t=2024-05-15T15:53:27.554397964Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=provisioning.datasources t=2024-05-15T15:53:27.554408384Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=provisioning.datasources t=2024-05-15T15:53:27.554538301Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=provisioning.alerting t=2024-05-15T15:53:27.576956139Z level=info msg="starting to provision alerting" 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=provisioning.alerting t=2024-05-15T15:53:27.576968063Z level=info msg="finished to provision alerting" 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=ngalert.state.manager t=2024-05-15T15:53:27.577159132Z level=info msg="Warming state cache for startup" 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=grafanaStorageLogger t=2024-05-15T15:53:27.577585976Z level=info msg="storage starting" 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=http.server t=2024-05-15T15:53:27.58573418Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=ngalert.state.manager t=2024-05-15T15:53:27.590908832Z level=info msg="State cache has been initialized" states=0 duration=13.748875ms 2024-05-15T15:53:27.881 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=ticker t=2024-05-15T15:53:27.590969711Z level=info msg=starting first_tick=2024-05-15T15:53:30Z 2024-05-15T15:53:27.882 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=ngalert.multiorg.alertmanager t=2024-05-15T15:53:27.590985469Z level=info msg="starting MultiOrg Alertmanager" 2024-05-15T15:53:27.882 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=sqlstore.transactions t=2024-05-15T15:53:27.598184574Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-05-15T15:53:27.882 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=sqlstore.transactions t=2024-05-15T15:53:27.609920147Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-05-15T15:53:27.882 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 15:53:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=sqlstore.transactions t=2024-05-15T15:53:27.62110773Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-05-15T15:53:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:27 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:27 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:27 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:27 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:27 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:28.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:27 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:29.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:28 smithi043 ceph-mon[24430]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:29.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:28 smithi043 ceph-mon[29127]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:29.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:28 smithi053 ceph-mon[25205]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:29.556 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:53:29.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:29 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:29.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:29 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:30.007 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:29 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:30.236 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:53:30.236 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (29s) 24s ago 22m 15.9M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:53:30.236 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 starting - - - - 2024-05-15T15:53:30.236 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 24s ago 21m 60.2M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:53:30.236 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (107s) 24s ago 27m 454M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:53:30.236 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (11m) 24s ago 28m 506M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:53:30.236 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (28m) 24s ago 29m 79.2M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:53:30.236 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (27m) 24s ago 27m 61.3M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:53:30.236 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (27m) 24s ago 27m 60.3M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (87s) 24s ago 23m 8334k - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (78s) 24s ago 23m 17.2M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (26m) 24s ago 26m 53.8M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (26m) 24s ago 26m 58.0M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (26m) 24s ago 26m 50.4M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (25m) 24s ago 25m 53.8M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (25m) 24s ago 25m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (24m) 24s ago 24m 51.3M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (24m) 24s ago 24m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (24m) 24s ago 24m 55.5M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (53s) 24s ago 23m 54.5M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (21m) 24s ago 21m 76.2M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:53:30.237 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (21m) 24s ago 21m 75.2M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:53:30.654 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:53:30.654 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:53:30.655 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-15T15:53:30.656 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:53:30.656 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:53:30.656 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:53:31.005 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:53:31.006 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T15:53:31.006 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:53:31.006 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-05-15T15:53:31.006 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [ 2024-05-15T15:53:31.006 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr" 2024-05-15T15:53:31.006 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T15:53:31.006 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "2/2 daemons upgraded", 2024-05-15T15:53:31.006 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading grafana daemons", 2024-05-15T15:53:31.006 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:53:31.006 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:53:31.007 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:30 smithi043 ceph-mon[24430]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:31.007 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:30 smithi043 ceph-mon[24430]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:31.007 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:30 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/451708475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:31.007 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:30 smithi043 ceph-mon[29127]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:31.007 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:30 smithi043 ceph-mon[29127]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:31.007 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:30 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/451708475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:30 smithi053 ceph-mon[25205]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:30 smithi053 ceph-mon[25205]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:30 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/451708475' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[24430]: from='client.15486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[24430]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[24430]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[24430]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:32.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[29127]: from='client.15486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[29127]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[29127]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[29127]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:31 smithi053 ceph-mon[25205]: from='client.15486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:31 smithi053 ceph-mon[25205]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:31 smithi053 ceph-mon[25205]: from='client.25411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:53:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:31 smithi053 ceph-mon[25205]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:33.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:33 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:33.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:33 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:33.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:33 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:33.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:33 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:33 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:33.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:33 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:34.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:34 smithi053 ceph-mon[25205]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:34.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:34 smithi043 ceph-mon[29127]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:34.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:34 smithi043 ceph-mon[24430]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:36.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:36 smithi043 ceph-mon[24430]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:36.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:36 smithi043 ceph-mon[29127]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:36.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:53:36] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.43.0" 2024-05-15T15:53:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:36 smithi053 ceph-mon[25205]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:38.524 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[24430]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:38.524 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:38.524 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:38.524 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:38.524 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:38.524 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:38.525 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:38.525 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:38.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[29127]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:38.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:38.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:38.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:38.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:38.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:38.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:38.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:38 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:38 smithi053 ceph-mon[25205]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:38.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:38.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:38.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:38.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:38 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[24430]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s 2024-05-15T15:53:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[24430]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:39.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:39.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[29127]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s 2024-05-15T15:53:39.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[29127]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:39.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:39.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:39.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:39 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:39 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:39 smithi053 ceph-mon[25205]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s 2024-05-15T15:53:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:39 smithi053 ceph-mon[25205]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:39 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:39 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:39 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:40.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:53:40.200+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:40.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:40.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:40.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:40.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:40.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:40.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:40.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:40.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:40.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:40.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:40.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:40.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:40.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:40.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:40.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:53:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:53:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:53:40.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:53:40.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:53:40.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:53:40.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:53:40.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T15:53:40.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:53:40.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:53:40.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:53:40.842 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:40.843 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:40.844 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:40.844 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:40.844 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:40.844 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:40.844 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:40 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:40.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:53:40.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:53:40.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:53:40.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:53:40.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:40.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:40.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:40.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:40.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:40.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:40 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:41.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:41.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:41.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:41.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:41.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:41.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:41.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:53:41.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:41.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:41.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:41.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:41.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:41.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:41.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:41.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:41.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:41.590 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:41.590 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:41.590 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:41.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:41.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:41.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:41.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:41.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:41.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:41.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:41.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:41.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:41.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:41.594 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:41.594 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[24430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[24430]: Upgrade: Finalizing container_image settings 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[24430]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[24430]: Upgrade: Complete! 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[24430]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[24430]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 984 B/s rd, 0 op/s 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:41.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[29127]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[29127]: Upgrade: Finalizing container_image settings 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[29127]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[29127]: Upgrade: Complete! 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[29127]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[29127]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 984 B/s rd, 0 op/s 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:41.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:41 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:41 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:41.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:41.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:41.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:41.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:41 smithi053 ceph-mon[25205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:41 smithi053 ceph-mon[25205]: Upgrade: Finalizing container_image settings 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:41 smithi053 ceph-mon[25205]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:41 smithi053 ceph-mon[25205]: Upgrade: Complete! 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:41 smithi053 ceph-mon[25205]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:41 smithi053 ceph-mon[25205]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 984 B/s rd, 0 op/s 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:41 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:41.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:41 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:42.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:53:42.136+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:42.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:53:42.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:42.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:42.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:42.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:42.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:42.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:42.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:42.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:42.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:42.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:42.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:42.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:42.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:42.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:42.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:42.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:42 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:42.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:42.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:42.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:42.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:42.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:42.838 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:42 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:42 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:53:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:42 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:42.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:42 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:53:42.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:42 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:42.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:42 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:53:42.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:42 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:53:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:43 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:43.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:43.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:53:43.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:43.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:43.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:43.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:43.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:43.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:43.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:43.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:43.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:43.840 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:43 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:43.840 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:43.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:53:43.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:43.844 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:43 smithi043 ceph-mon[24430]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:43 smithi043 ceph-mon[24430]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:43 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:43 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:43.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:43 smithi043 ceph-mon[29127]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s 2024-05-15T15:53:43.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:43 smithi043 ceph-mon[29127]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:53:43.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:43 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:43.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:43 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:43 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:43.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:43.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:43.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:43.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:43.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:43.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:43.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:43.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:43 smithi053 ceph-mon[25205]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s 2024-05-15T15:53:43.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:43 smithi053 ceph-mon[25205]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:53:43.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:43 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:53:43.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:43 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:53:44.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:53:44.002+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:44.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:44.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:44.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:44.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:44.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:44.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:53:44.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:44.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:44.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:45 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:45.336 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:45.336 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:45.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:45.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:45.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:45.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:45.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:45.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:45.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:45.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:45.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:45.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:45.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:45.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:45.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:45.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:45.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:53:45.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:45.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:53:45.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:45.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:53:45.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:45.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:45.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:45.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:45.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:45.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:45.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:45.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:45.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:45.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:45.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:45.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:45.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:45.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:45.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:45.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:45.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:45.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:45.343 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:45 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:45.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:45.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:45.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:45.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:45.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:45.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:45.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:45.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:45.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:45.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:45.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:45.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:45.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:45.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:45.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:45.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:45.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.347 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.347 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:45.347 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:45.347 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:45.347 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:45.347 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:45.347 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:45.347 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:45.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:45.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:53:45.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:45.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:53:45.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:45.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:53:45.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:45.349 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:45.349 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:45.349 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:45.349 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:45.349 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:45.349 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:45.350 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:45.350 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:45.350 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.350 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.350 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:53:45.350 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:45.351 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:45.351 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:45.351 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:53:45.351 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:45.351 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:45.351 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:45.351 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:45.352 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:45.352 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:45.352 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:45.352 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.352 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.352 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:45 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:45.352 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:45 smithi043 ceph-mon[24430]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-05-15T15:53:45.352 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:45.353 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:45.353 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:45.353 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.353 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.353 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:45 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:45.353 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:45 smithi043 ceph-mon[29127]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-05-15T15:53:45.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:45 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:53:45.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:53:45.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:53:45.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:53:45.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:53:45.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:53:45.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:53:45.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:53:45.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:53:45.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:53:45.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:45 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:53:45.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:45 smithi053 ceph-mon[25205]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-05-15T15:53:46.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:53:46] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.43.0" 2024-05-15T15:53:47.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:47 smithi043 ceph-mon[24430]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-05-15T15:53:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:47 smithi043 ceph-mon[29127]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-05-15T15:53:47.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:47 smithi053 ceph-mon[25205]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-05-15T15:53:49.799 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:49 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:49.800 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:49 smithi043 ceph-mon[24430]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-05-15T15:53:49.800 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:49 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:49.800 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:49 smithi043 ceph-mon[29127]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-05-15T15:53:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:49 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:49 smithi053 ceph-mon[25205]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-05-15T15:53:51.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:51 smithi043 ceph-mon[24430]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:53:51.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:51 smithi043 ceph-mon[29127]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:53:51.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:51 smithi053 ceph-mon[25205]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:53:53.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:53 smithi043 ceph-mon[24430]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:53:53.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:53 smithi043 ceph-mon[29127]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:53:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:53 smithi053 ceph-mon[25205]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:53:54.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:54.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:54 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:53:55.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:55 smithi043 ceph-mon[24430]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:55.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:55 smithi043 ceph-mon[29127]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:55 smithi053 ceph-mon[25205]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:56.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:53:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:53:56] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.43.0" 2024-05-15T15:53:57.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:57 smithi043 ceph-mon[24430]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:57.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:57 smithi043 ceph-mon[29127]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:57.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:57 smithi053 ceph-mon[25205]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:53:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:59 smithi043 ceph-mon[24430]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:59.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:53:59 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:59 smithi043 ceph-mon[29127]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:59.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:53:59 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:53:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:59 smithi053 ceph-mon[25205]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:53:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:53:59 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:01.749 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:01 smithi043 ceph-mon[24430]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:01.749 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:01 smithi043 ceph-mon[29127]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:01.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:01 smithi053 ceph-mon[25205]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:02.024 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-05-15T15:54:02.645 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:02 smithi043 ceph-mon[24430]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:02.645 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:02 smithi043 ceph-mon[29127]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:02.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:02 smithi053 ceph-mon[25205]: from='client.15510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:03.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:03 smithi043 ceph-mon[29127]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:03 smithi043 ceph-mon[24430]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:03.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:03 smithi053 ceph-mon[25205]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:04.108 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:54:04.673 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:04 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/4141958505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:04.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:04 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/4141958505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:04.700 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-05-15T15:54:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:04 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/4141958505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:05.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:05 smithi043 ceph-mon[24430]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:05.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:05 smithi043 ceph-mon[29127]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:05 smithi053 ceph-mon[25205]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:06.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:54:06] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.43.0" 2024-05-15T15:54:06.765 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)" 2024-05-15T15:54:07.399 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-05-15T15:54:07.718 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:07 smithi043 ceph-mon[24430]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:07.718 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:07 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3167135052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:07.718 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:07 smithi043 ceph-mon[29127]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:07.718 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:07 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3167135052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:07 smithi053 ceph-mon[25205]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:07.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:07 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3167135052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:09.491 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:54:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:09 smithi043 ceph-mon[24430]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:09 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:09 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:09 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/3955889657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:09.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:09 smithi043 ceph-mon[29127]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:09.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:09 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:09.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:09 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:09.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:09 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/3955889657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:09 smithi053 ceph-mon[25205]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:09 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:09 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:09 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/3955889657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:10.213 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-05-15T15:54:11.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:11 smithi043 ceph-mon[24430]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:11.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:11 smithi043 ceph-mon[29127]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:11 smithi053 ceph-mon[25205]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:13 smithi043 ceph-mon[24430]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:13 smithi043 ceph-mon[24430]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:13 smithi043 ceph-mon[29127]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:13.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:13 smithi043 ceph-mon[29127]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:13 smithi053 ceph-mon[25205]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:13.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:13 smithi053 ceph-mon[25205]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:14.294 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:54:14.895 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade status' 2024-05-15T15:54:15.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:15 smithi043 ceph-mon[24430]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:15.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:15 smithi043 ceph-mon[29127]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:15.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:15 smithi053 ceph-mon[25205]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:16.461 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:54:16] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.43.0" 2024-05-15T15:54:16.910 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:54:16.910 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": null, 2024-05-15T15:54:16.910 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": false, 2024-05-15T15:54:16.910 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "", 2024-05-15T15:54:16.910 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:54:16.911 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": null, 2024-05-15T15:54:16.911 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "", 2024-05-15T15:54:16.911 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:54:16.911 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:54:17.591 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T15:54:17.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:17 smithi043 ceph-mon[24430]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:17.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:17 smithi043 ceph-mon[29127]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:17.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:17 smithi053 ceph-mon[25205]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:18.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:18 smithi043 ceph-mon[29127]: from='client.25447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:18.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:18 smithi043 ceph-mon[24430]: from='client.25447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:18 smithi053 ceph-mon[25205]: from='client.25447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:19.681 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:54:19.682 INFO:teuthology.orchestra.run.smithi043.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:54:19.682 INFO:teuthology.orchestra.run.smithi043.stdout: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:54:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:19 smithi043 ceph-mon[24430]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:19.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:19 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:19 smithi043 ceph-mon[29127]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:19.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:19 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:19 smithi053 ceph-mon[25205]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:19.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:19 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:20.259 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-05-15T15:54:20.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:20 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/750099919' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:54:20.829 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:20 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/750099919' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:54:20.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:20 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/750099919' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:54:21.800 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:21 smithi043 ceph-mon[24430]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:21.800 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:21 smithi043 ceph-mon[29127]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:21.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:21 smithi053 ceph-mon[25205]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:23 smithi053 ceph-mon[25205]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:23 smithi053 ceph-mon[25205]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:23 smithi053 ceph-mon[25205]: from='client.25465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "mon", "hosts": "smithi053", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:23.947 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:23 smithi043 ceph-mon[29127]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:23.947 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:23 smithi043 ceph-mon[29127]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:23.947 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:23 smithi043 ceph-mon[29127]: from='client.25465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "mon", "hosts": "smithi053", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:23.947 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:23 smithi043 ceph-mon[24430]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:23.947 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:23 smithi043 ceph-mon[24430]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:23.947 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:23 smithi043 ceph-mon[24430]: from='client.25465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "mon", "hosts": "smithi053", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:24.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:24 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:24.920 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:24 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:24.920 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:24 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:24.964 INFO:teuthology.orchestra.run.smithi043.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:54:25.619 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-15T15:54:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:25 smithi053 ceph-mon[25205]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:25 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:25 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:25.942 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:25 smithi043 ceph-mon[24430]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:25.942 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:25 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:25.942 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:25 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:25.942 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:25 smithi043 ceph-mon[29127]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:25.942 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:25 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:25.942 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:25 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:26.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:54:26] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.43.0" 2024-05-15T15:54:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:26 smithi053 ceph-mon[25205]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:54:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:26 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:26 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:26 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:26 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:26.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:26 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:26.907 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[29127]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:54:26.907 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:26.907 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:26.907 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:26.907 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:26.907 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:26.908 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[24430]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:54:26.908 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:26.908 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:26.908 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:26.908 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:26.908 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:26 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:27.608 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:54:27.789 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:27 smithi043 ceph-mon[24430]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:27.789 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:27 smithi043 ceph-mon[24430]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:27.790 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:27 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:27.790 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:27 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:27.790 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:27 smithi043 ceph-mon[29127]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:27.790 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:27 smithi043 ceph-mon[29127]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:27.790 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:27 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:27.790 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:27 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:27 smithi053 ceph-mon[25205]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:27 smithi053 ceph-mon[25205]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:27.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:27 smithi053 ceph-mon[25205]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:27.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:27 smithi053 ceph-mon[25205]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:28.297 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:54:28.297 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (87s) 82s ago 23m 15.9M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:54:28.297 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (62s) 57s ago 23m 53.6M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:54:28.297 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 82s ago 22m 60.2M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:54:28.297 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (2m) 57s ago 28m 454M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:54:28.297 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (12m) 82s ago 29m 506M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:54:28.297 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (29m) 82s ago 29m 79.2M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:54:28.297 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (28m) 57s ago 28m 61.3M 2048M 17.2.0 e1d6a67b021e 005e4f2fbae3 2024-05-15T15:54:28.298 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (28m) 82s ago 28m 60.3M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:54:28.298 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (2m) 82s ago 24m 8334k - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:54:28.298 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (2m) 57s ago 24m 17.4M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:54:28.298 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (27m) 82s ago 27m 53.8M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:54:28.298 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (27m) 82s ago 27m 58.0M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:54:28.298 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (27m) 82s ago 27m 50.4M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:54:28.298 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (26m) 82s ago 26m 53.8M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:54:28.298 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (26m) 57s ago 26m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:54:28.298 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (25m) 57s ago 25m 51.4M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:54:28.298 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (25m) 57s ago 25m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:54:28.299 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (25m) 57s ago 25m 55.5M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:54:28.299 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (111s) 57s ago 24m 60.4M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:54:28.299 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (22m) 82s ago 22m 76.2M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:54:28.299 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (22m) 57s ago 22m 75.3M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:54:28.731 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:28 smithi043 ceph-mon[24430]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:28.731 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:28 smithi043 ceph-mon[29127]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:54:28.732 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:54:28.733 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:54:28.733 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:54:28.733 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-15T15:54:28.733 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:54:28.733 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:54:28.733 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:54:28.733 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:54:28.588+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:54:28.734 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:28.735 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:28.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:28.737 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:28.738 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:28.738 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:28.738 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:28.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:28 smithi053 ceph-mon[25205]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:29.089 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:54:29.089 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", 2024-05-15T15:54:29.089 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:54:29.089 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi053", 2024-05-15T15:54:29.089 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:54:29.089 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "", 2024-05-15T15:54:29.089 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 image", 2024-05-15T15:54:29.089 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:54:29.089 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:54:29.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[29127]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:29.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[29127]: from='client.15570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:29.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:29.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[29127]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:29.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:29.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:29.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:29.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:29.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:29.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:29.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:29.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:29.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:29.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:29.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:29.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:29.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:29.846 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:29.847 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:29.847 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:29.847 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:29.847 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:29.847 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.847 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[29127]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:54:29.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/2672887556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:29.848 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[24430]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:29.848 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[24430]: from='client.15570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:29.848 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:29.848 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[24430]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:29.848 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:29.848 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:29.848 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:29.848 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:29.848 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:29.849 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:54:29.850 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:29.851 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[24430]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:54:29.852 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:29 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/2672887556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:29.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:29 smithi053 ceph-mon[25205]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:29 smithi053 ceph-mon[25205]: from='client.15570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:29 smithi053 ceph-mon[25205]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:29 smithi053 ceph-mon[25205]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:29 smithi053 ceph-mon[25205]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:29.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:29.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:29.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:29.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:29.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:29.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:29.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:29.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:29.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:29.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:29.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:29.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:29 smithi053 ceph-mon[25205]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:54:29.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:29 smithi053 ceph-mon[25205]: from='client.? 172.21.15.43:0/2672887556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:30.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:30 smithi053 ceph-mon[25205]: from='client.25495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:30.952 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:30 smithi043 ceph-mon[29127]: from='client.25495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:30.952 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:30 smithi043 ceph-mon[24430]: from='client.25495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:54:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-05-15T15:54:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:54:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:54:31.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all mgr 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[24430]: Upgrade: It appears safe to stop mon.b 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:31.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:31.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all mgr 2024-05-15T15:54:31.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:31.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-15T15:54:31.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-15T15:54:31.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:31 smithi043 ceph-mon[29127]: Upgrade: It appears safe to stop mon.b 2024-05-15T15:54:31.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-05-15T15:54:31.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:31.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:54:31.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:54:31.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:31.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:31.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: Upgrade: Setting container_image for all mgr 2024-05-15T15:54:31.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:31.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-15T15:54:31.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-05-15T15:54:31.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:31 smithi053 ceph-mon[25205]: Upgrade: It appears safe to stop mon.b 2024-05-15T15:54:33.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:33 smithi053 ceph-mon[25205]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:33.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:33 smithi053 ceph-mon[25205]: Upgrade: Updating mon.b 2024-05-15T15:54:33.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:33 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:33.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:33 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:54:33.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:33 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:54:33.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:33 smithi053 ceph-mon[25205]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:33.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:33 smithi053 ceph-mon[25205]: Deploying daemon mon.b on smithi053 2024-05-15T15:54:33.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[24430]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:33.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[24430]: Upgrade: Updating mon.b 2024-05-15T15:54:33.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:33.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:54:33.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:54:33.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:33.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[24430]: Deploying daemon mon.b on smithi053 2024-05-15T15:54:33.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[29127]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:33.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[29127]: Upgrade: Updating mon.b 2024-05-15T15:54:33.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:33.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:54:33.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:54:33.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:33.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:33 smithi043 ceph-mon[29127]: Deploying daemon mon.b on smithi053 2024-05-15T15:54:34.349 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:34 smithi053 systemd[1]: Stopping Ceph mon.b for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:54:34.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b[25198]: 2024-05-15T15:54:34.345+0000 7fae88dcf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T15:54:34.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b[25198]: 2024-05-15T15:54:34.345+0000 7fae88dcf700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-05-15T15:54:34.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:34 smithi053 podman[67057]: 2024-05-15 15:54:34.376959531 +0000 UTC m=+0.153839369 container died 005e4f2fbae38b90445cfe84d372a44f6237e30370b33d9d5e0056f4a1a55e85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, name=centos-stream, version=8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, distribution-scope=public, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc.) 2024-05-15T15:54:35.081 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:34 smithi053 podman[67057]: 2024-05-15 15:54:34.820699509 +0000 UTC m=+0.597579347 container cleanup 005e4f2fbae38b90445cfe84d372a44f6237e30370b33d9d5e0056f4a1a55e85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., architecture=x86_64, maintainer=Guillaume Abrioux , version=8, GIT_CLEAN=True, distribution-scope=public, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0) 2024-05-15T15:54:35.082 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:34 smithi053 bash[67057]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b 2024-05-15T15:54:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 podman[67070]: 2024-05-15 15:54:35.080297431 +0000 UTC m=+0.705227100 container remove 005e4f2fbae38b90445cfe84d372a44f6237e30370b33d9d5e0056f4a1a55e85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, io.openshift.expose-services=, distribution-scope=public, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, vcs-type=git, RELEASE=HEAD, architecture=x86_64, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream) 2024-05-15T15:54:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.b.service: Deactivated successfully. 2024-05-15T15:54:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 systemd[1]: Stopped Ceph mon.b for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:54:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.b.service: Consumed 19.264s CPU time. 2024-05-15T15:54:35.791 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 systemd[1]: Starting Ceph mon.b for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:54:35.791 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 podman[67201]: 2024-05-15 15:54:35.68179004 +0000 UTC m=+0.108669119 container create 0bcacc4488236baf14a65ab4165190a0858f3ed780781b9a96474a3da7237749 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-05-15T15:54:35.791 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 podman[67201]: 2024-05-15 15:54:35.591087962 +0000 UTC m=+0.017967048 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 podman[67201]: 2024-05-15 15:54:35.90966759 +0000 UTC m=+0.336546670 container init 0bcacc4488236baf14a65ab4165190a0858f3ed780781b9a96474a3da7237749 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 podman[67201]: 2024-05-15 15:54:35.913062906 +0000 UTC m=+0.339941986 container start 0bcacc4488236baf14a65ab4165190a0858f3ed780781b9a96474a3da7237749 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: set uid:gid to 167:167 (ceph:ceph) 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev), process ceph-mon, pid 2 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: pidfile_write: ignore empty --pid-file 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: load: jerasure load: lrc 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: RocksDB version: 7.9.2 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Git sha 0 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Compile date 2024-05-15 11:18:19 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: DB SUMMARY 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: DB Session ID: IEGDCUJ2RHO3N7OAH369 2024-05-15T15:54:36.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: CURRENT file: CURRENT 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: IDENTITY file: IDENTITY 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2814 Bytes 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 3444422 ; 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.error_if_exists: 0 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.create_if_missing: 0 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.paranoid_checks: 1 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.env: 0x5620ace37c60 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.fs: PosixFileSystem 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.info_log: 0x5620ae5e3ac0 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_file_opening_threads: 16 2024-05-15T15:54:36.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.statistics: (nil) 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.use_fsync: 0 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_log_file_size: 0 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.keep_log_file_num: 1000 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.recycle_log_file_num: 0 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.allow_fallocate: 1 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.allow_mmap_reads: 0 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.allow_mmap_writes: 0 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.use_direct_reads: 0 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.create_missing_column_families: 0 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.db_log_dir: 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.wal_dir: 2024-05-15T15:54:36.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.advise_random_on_open: 1 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.db_write_buffer_size: 0 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.write_buffer_manager: 0x5620ae5b61e0 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.rate_limiter: (nil) 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-15T15:54:36.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.wal_recovery_mode: 2 2024-05-15T15:54:36.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.enable_thread_tracking: 0 2024-05-15T15:54:36.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.enable_pipelined_write: 0 2024-05-15T15:54:36.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.unordered_write: 0 2024-05-15T15:54:36.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-15T15:54:36.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-15T15:54:36.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-15T15:54:36.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-15T15:54:36.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.row_cache: None 2024-05-15T15:54:36.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.wal_filter: None 2024-05-15T15:54:36.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-15T15:54:36.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.allow_ingest_behind: 0 2024-05-15T15:54:36.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.two_write_queues: 0 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.manual_wal_flush: 0 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.wal_compression: 0 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.atomic_flush: 0 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.log_readahead_size: 0 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.best_efforts_recovery: 0 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.allow_data_in_errors: 0 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.db_host_id: __hostname__ 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-15T15:54:36.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_background_jobs: 2 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_background_compactions: -1 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_subcompactions: 1 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_total_wal_size: 0 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_open_files: -1 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bytes_per_sync: 0 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-15T15:54:36.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_readahead_size: 0 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_background_flushes: -1 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Compression algorithms supported: 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: kZSTD supported: 0 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: kXpressCompression supported: 0 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: kLZ4HCCompression supported: 1 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: kZlibCompression supported: 1 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: kSnappyCompression supported: 1 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: kLZ4Compression supported: 1 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: kBZip2Compression supported: 0 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-05-15T15:54:36.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.merge_operator: 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_filter: None 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_filter_factory: None 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.sst_partitioner_factory: None 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5620ae5e3bc0) 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout: cache_index_and_filter_blocks: 1 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-15T15:54:36.140 INFO:journalctl@ceph.mon.b.smithi053.stdout: pin_top_level_index_and_filter: 1 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: index_type: 0 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: data_block_index_type: 0 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: index_shortening: 1 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: checksum: 4 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: no_block_cache: 0 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_cache: 0x5620ae5e7090 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_cache_name: BinnedLRUCache 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_cache_options: 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: capacity : 536870912 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: num_shard_bits : 4 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: strict_capacity_limit : 0 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: high_pri_pool_ratio: 0.000 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_cache_compressed: (nil) 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: persistent_cache: (nil) 2024-05-15T15:54:36.141 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_size: 4096 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_size_deviation: 10 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_restart_interval: 16 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: index_block_restart_interval: 1 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: metadata_block_size: 4096 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: partition_filters: 0 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_delta_encoding: 1 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: filter_policy: bloomfilter 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: whole_key_filtering: 1 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: verify_compression: 0 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: read_amp_bytes_per_bit: 0 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: format_version: 5 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: enable_index_compression: 1 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: block_align: 0 2024-05-15T15:54:36.142 INFO:journalctl@ceph.mon.b.smithi053.stdout: max_auto_readahead_size: 262144 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: prepopulate_block_cache: 0 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: initial_auto_readahead_size: 8192 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout: num_file_reads_for_auto_readahead: 2 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.write_buffer_size: 33554432 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_write_buffer_number: 2 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compression: NoCompression 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bottommost_compression: Disabled 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.prefix_extractor: nullptr 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.num_levels: 7 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-15T15:54:36.143 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compression_opts.level: 32767 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compression_opts.strategy: 0 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-15T15:54:36.144 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compression_opts.enabled: false 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.target_file_size_base: 67108864 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-15T15:54:36.145 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.arena_block_size: 1048576 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.disable_auto_compactions: 0 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-15T15:54:36.146 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.table_properties_collectors: 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.inplace_update_support: 0 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.bloom_locality: 0 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.max_successive_merges: 0 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.paranoid_file_checks: 0 2024-05-15T15:54:36.147 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.force_consistency_checks: 1 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.report_bg_io_stats: 0 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.ttl: 2592000 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.enable_blob_files: false 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.min_blob_size: 0 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.blob_file_size: 268435456 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.blob_file_starting_level: 0 2024-05-15T15:54:36.148 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28812, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 39058f3a-6eb1-46a9-9f2d-d7f7ebb93c16 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715788475943767, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715788475953210, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 2129095, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28817, "largest_seqno": 30771, "table_properties": {"data_size": 2121475, "index_size": 4744, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1861, "raw_key_size": 17248, "raw_average_key_size": 23, "raw_value_size": 2105916, "raw_average_value_size": 2880, "num_data_blocks": 214, "num_entries": 731, "num_filter_entries": 731, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1715788475, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "39058f3a-6eb1-46a9-9f2d-d7f7ebb93c16", "db_session_id": "IEGDCUJ2RHO3N7OAH369", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715788475953308, "job": 1, "event": "recovery_finished"} 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5620ae6d8000 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: DB pointer 0x5620ae6c2000 2024-05-15T15:54:36.149 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** DB Stats ** 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** Compaction Stats [default] ** 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-15T15:54:36.150 INFO:journalctl@ceph.mon.b.smithi053.stdout: L0 1/0 2.03 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 511.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: L6 1/0 9.01 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: Sum 2/0 11.04 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 511.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 511.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** Compaction Stats [default] ** 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 511.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-05-15T15:54:36.151 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: Cumulative compaction: 0.00 GB write, 169.83 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: Interval compaction: 0.00 GB write, 169.83 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: Block cache BinnedLRUCache@0x5620ae5e7090#2 capacity: 512.00 MB usage: 7.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: starting mon.b rank 2 at public addrs [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] at bind addrs [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: mon.b@-1(???) e3 preinit fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: mon.b@-1(???).mds e1 new map 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: mon.b@-1(???).mds e1 print_map 2024-05-15T15:54:36.152 INFO:journalctl@ceph.mon.b.smithi053.stdout: e1 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout: legacy client fscid: -1 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout: 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout: No filesystems configured 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: mon.b@-1(???).osd e106 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: mon.b@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: mon.b@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: mon.b@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:35 smithi053 ceph-mon[67220]: mon.b@-1(???).paxosservice(auth 1..25) refresh upgraded, format 0 -> 3 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:36 smithi053 bash[67201]: 0bcacc4488236baf14a65ab4165190a0858f3ed780781b9a96474a3da7237749 2024-05-15T15:54:36.153 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:36 smithi053 systemd[1]: Started Ceph mon.b for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:54:36.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:36 smithi043 ceph-mon[24430]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:36.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:36 smithi043 ceph-mon[29127]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:36.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:54:36] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.43.0" 2024-05-15T15:54:36.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:36 smithi053 ceph-mon[67220]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:37.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: mon.b calling monitor election 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: mon.c calling monitor election 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: mon.a calling monitor election 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: monmap e3: 3 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],b=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: fsmap 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: osdmap e106: 8 total, 8 up, 8 in 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: mgrmap e42: y(active, since 3m), standbys: x 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:54:37.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:37.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:37.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:37 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:37.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:54:37.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: mon.b calling monitor election 2024-05-15T15:54:37.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: mon.c calling monitor election 2024-05-15T15:54:37.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: mon.a calling monitor election 2024-05-15T15:54:37.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: monmap e3: 3 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],b=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: fsmap 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: osdmap e106: 8 total, 8 up, 8 in 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: mgrmap e42: y(active, since 3m), standbys: x 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:37.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: mon.b calling monitor election 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: mon.c calling monitor election 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: mon.a calling monitor election 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: monmap e3: 3 mons at {a=[v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0],b=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],c=[v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0]} 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: fsmap 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: osdmap e106: 8 total, 8 up, 8 in 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: mgrmap e42: y(active, since 3m), standbys: x 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:37.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:37 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:39.501 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:39 smithi053 ceph-mon[67220]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:39.501 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:39 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:39.502 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:39 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:39 smithi043 ceph-mon[24430]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:39 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:39.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:39 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:39.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:39 smithi043 ceph-mon[29127]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:39.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:39 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:39.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:39 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:41.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:40 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:54:40.962+0000 7ffac1bf4700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-05-15T15:54:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:41 smithi043 ceph-mon[24430]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:41 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:41 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:41 smithi043 ceph-mon[29127]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:41.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:41 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:41.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:41 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:41 smithi053 ceph-mon[67220]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:41 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:41 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:43.037 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:42 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:43.037 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:42 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:43.037 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:42 smithi053 ceph-mon[67220]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:43.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:42 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:43.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:42 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:43.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:42 smithi043 ceph-mon[24430]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:43.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:42 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:43.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:42 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:43.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:42 smithi043 ceph-mon[29127]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:54:45.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:44 smithi053 ceph-mon[67220]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:45.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:44 smithi043 ceph-mon[24430]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:45.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:44 smithi043 ceph-mon[29127]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:54:46.449 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:54:46] "GET /metrics HTTP/1.1" 200 37926 "" "Prometheus/2.43.0" 2024-05-15T15:54:46.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: Detected new or changed devices on smithi053 2024-05-15T15:54:46.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:46.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: Detected new or changed devices on smithi053 2024-05-15T15:54:46.450 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:46.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:46.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:46.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:46.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:46.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:46.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:46.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:46.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:46.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:46.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:46 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: Detected new or changed devices on smithi053 2024-05-15T15:54:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:46 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:47.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:54:47.211+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:47.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:47.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:47.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:47.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:47.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:47.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:47.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:47.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:47.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:47.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:47.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:47.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:54:47.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:54:47.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T15:54:47.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:54:47.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:54:47.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T15:54:47.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:54:47.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:54:47.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:54:47.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:54:47.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:47.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:47.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:47.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:47 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.839 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:47.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.839 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.840 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.841 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.841 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.841 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.841 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.841 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.841 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.841 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.841 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.841 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T15:54:47.842 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:54:47.843 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:54:47.843 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T15:54:47.843 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.843 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:54:47.843 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T15:54:47.843 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:54:47.843 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:54:47.843 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T15:54:47.843 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:54:47.843 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:54:47.844 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:54:47.844 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-05-15T15:54:47.844 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:54:47.844 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:54:47.844 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.844 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:54:47.844 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:54:47.844 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T15:54:47.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:54:47.846 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:54:47.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:54:47.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:54:47.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:54:47.847 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:47.848 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:47.848 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:47.848 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:47 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:48.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:48.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:48.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:48.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:48.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:48.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:48.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:54:48.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:48.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:48.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all crash 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mds 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all nfs 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all node-exporter 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all prometheus 2024-05-15T15:54:48.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all alertmanager 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all grafana 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all loki 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all promtail 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Finalizing container_image settings 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Upgrade: Complete! 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:48.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:48 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:48.677 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:48.677 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:48.677 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:48.677 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:48.677 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:48.677 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.677 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.677 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.678 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:48.679 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.680 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:48.681 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:48.681 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:48.681 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:48.681 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:48.681 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:48.681 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:48.681 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:48.681 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:48.682 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:48.682 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:48.682 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:48.682 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:48.682 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:48.682 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.682 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.682 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:48.682 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:48.683 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:48.684 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:48.685 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all crash 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all mds 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T15:54:48.686 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all nfs 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all node-exporter 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all prometheus 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all alertmanager 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all grafana 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all loki 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all promtail 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Finalizing container_image settings 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Upgrade: Complete! 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:48.687 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all crash 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all mds 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T15:54:48.688 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all nfs 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all node-exporter 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all prometheus 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all alertmanager 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all grafana 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all loki 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all promtail 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Finalizing container_image settings 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Upgrade: Complete! 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:48.689 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:48.690 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:48.690 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:48 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:49.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:54:49.133+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:49.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:54:49.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:49.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:49.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:49.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:49.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:49.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:54:49.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:49.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:54:49.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:49.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:49.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:49.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:54:49.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:49.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:49.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:49.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:49.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:49.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:49.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:49.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:49.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:49 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:49 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:49 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:49 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:49 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:49 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:49 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:49 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:49.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:54:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:54:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:54:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:54:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:49 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:54:50.498 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:50 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:50.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:50.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:50.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:50.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:50.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:54:50.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:50.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:50.502 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:50.502 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:50.502 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:50.502 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:50.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:50 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:50.502 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:50.502 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:50.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:54:50.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:50.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:50 smithi043 ceph-mon[24430]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 695 B/s rd, 0 op/s 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:50 smithi043 ceph-mon[24430]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:50 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:50 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.507 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:50 smithi043 ceph-mon[29127]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 695 B/s rd, 0 op/s 2024-05-15T15:54:50.508 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:50 smithi043 ceph-mon[29127]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-05-15T15:54:50.508 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:50 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:50.508 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:50 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:50 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:50.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:50.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:50.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:50.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:50 smithi053 ceph-mon[67220]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 695 B/s rd, 0 op/s 2024-05-15T15:54:50.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:50 smithi053 ceph-mon[67220]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-05-15T15:54:50.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:50 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:54:50.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:50 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:54:51.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:54:51.050+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:51.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:54:51.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:51.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:51.283 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:51.284 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:51.285 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:51.286 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:51.287 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:51.288 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:51.288 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:51.288 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:52 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:52.837 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:52.838 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:52.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:52.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:52 smithi043 ceph-mon[24430]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:52 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:52.841 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:52.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:52.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:52.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.845 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:52 smithi043 ceph-mon[29127]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-05-15T15:54:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:52 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:52.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:54:52.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:54:52.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:54:52.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:54:52.885 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:52 smithi053 ceph-mon[67220]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-05-15T15:54:54.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:54 smithi043 ceph-mon[24430]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 657 B/s rd, 0 op/s 2024-05-15T15:54:54.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:54 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:54 smithi043 ceph-mon[29127]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 657 B/s rd, 0 op/s 2024-05-15T15:54:54.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:54 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:54 smithi053 ceph-mon[67220]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 657 B/s rd, 0 op/s 2024-05-15T15:54:54.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:54 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:54:56.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:56 smithi043 ceph-mon[24430]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:56.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:56 smithi043 ceph-mon[29127]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:56.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:54:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:54:56] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-05-15T15:54:56.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:56 smithi053 ceph-mon[67220]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:54:58.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:58 smithi043 ceph-mon[24430]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:58.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:58 smithi043 ceph-mon[29127]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:58.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:58 smithi053 ceph-mon[67220]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:54:59.779 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:54:59 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:59.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:54:59 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:54:59.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:54:59 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:00.099 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T15:55:00.742 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:00 smithi043 ceph-mon[24430]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:55:00.742 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:00 smithi043 ceph-mon[24430]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:00.742 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:00 smithi043 ceph-mon[29127]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:55:00.743 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:00 smithi043 ceph-mon[29127]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:00 smithi053 ceph-mon[67220]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:55:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:00 smithi053 ceph-mon[67220]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:02.127 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:55:02.127 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (2m) 116s ago 24m 15.9M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:55:02.127 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (95s) 21s ago 23m 74.6M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:55:02.127 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 116s ago 23m 60.2M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:55:02.127 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (3m) 21s ago 28m 456M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:55:02.127 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (13m) 116s ago 30m 506M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:55:02.127 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (30m) 116s ago 30m 79.2M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:55:02.127 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (26s) 21s ago 28m 20.0M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:55:02.127 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (29m) 116s ago 29m 60.3M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:55:02.127 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (2m) 116s ago 24m 8334k - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (2m) 21s ago 24m 16.8M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (28m) 116s ago 28m 53.8M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (28m) 116s ago 28m 58.0M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (27m) 116s ago 27m 50.4M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (27m) 116s ago 27m 53.8M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (26m) 21s ago 26m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (26m) 21s ago 26m 51.6M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (26m) 21s ago 26m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (25m) 21s ago 25m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (2m) 21s ago 24m 66.6M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (23m) 116s ago 23m 76.2M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:55:02.128 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (23m) 21s ago 23m 75.6M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:55:02.746 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:02 smithi043 ceph-mon[24430]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:02.746 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:02 smithi043 ceph-mon[29127]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:02.770 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-05-15T15:55:02.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:02 smithi053 ceph-mon[67220]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:03.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:03 smithi043 ceph-mon[24430]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:03 smithi043 ceph-mon[29127]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:03.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:03 smithi053 ceph-mon[67220]: from='client.15600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:04.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:04 smithi043 ceph-mon[24430]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:04.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:04 smithi043 ceph-mon[29127]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:04.842 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:55:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:04 smithi053 ceph-mon[67220]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:05.505 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade status' 2024-05-15T15:55:05.823 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:05 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1888780457' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:05.823 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:05 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1888780457' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:05.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:05 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1888780457' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:06.493 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:55:06] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-05-15T15:55:06.793 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:06 smithi043 ceph-mon[24430]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:06.793 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:06 smithi043 ceph-mon[29127]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:06.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:06 smithi053 ceph-mon[67220]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:07.519 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:55:07.519 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": null, 2024-05-15T15:55:07.519 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": false, 2024-05-15T15:55:07.520 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "", 2024-05-15T15:55:07.520 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:55:07.520 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": null, 2024-05-15T15:55:07.520 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "", 2024-05-15T15:55:07.520 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:55:07.520 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:55:08.164 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T15:55:08.738 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:08 smithi043 ceph-mon[24430]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:08.738 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:08 smithi043 ceph-mon[24430]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:08.738 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:08 smithi043 ceph-mon[29127]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:08.739 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:08 smithi043 ceph-mon[29127]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:08 smithi053 ceph-mon[67220]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:08 smithi053 ceph-mon[67220]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:09.655 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:09 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:09.655 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:09 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:55:09.656 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:09 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:09.656 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:09 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:55:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:09 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:09 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:55:10.172 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:55:10.173 INFO:teuthology.orchestra.run.smithi043.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:55:10.173 INFO:teuthology.orchestra.run.smithi043.stdout: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:55:10.717 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-05-15T15:55:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:10 smithi043 ceph-mon[24430]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:10.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:10 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/1841404903' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:55:10.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:10 smithi043 ceph-mon[29127]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:10.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:10 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/1841404903' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:55:10.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:10 smithi053 ceph-mon[67220]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:10.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:10 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1841404903' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:55:12.767 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:12 smithi043 ceph-mon[24430]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:12.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:12 smithi043 ceph-mon[29127]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:12.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:12 smithi053 ceph-mon[67220]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:14 smithi043 ceph-mon[24430]: from='client.15624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:14 smithi043 ceph-mon[24430]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:14 smithi043 ceph-mon[24430]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "mon", "hosts": "smithi043", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:14.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:14 smithi043 ceph-mon[29127]: from='client.15624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:14.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:14 smithi043 ceph-mon[29127]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:14.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:14 smithi043 ceph-mon[29127]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "mon", "hosts": "smithi043", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:14 smithi053 ceph-mon[67220]: from='client.15624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:14 smithi053 ceph-mon[67220]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:55:14.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:14 smithi053 ceph-mon[67220]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "mon", "hosts": "smithi043", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:15.270 INFO:teuthology.orchestra.run.smithi043.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:55:15.941 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-15T15:55:16.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:55:16] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-05-15T15:55:16.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:16.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:55:16.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:16.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1010 B/s rd, 0 op/s 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:16.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1010 B/s rd, 0 op/s 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:16.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:16 smithi043 ceph-mon[24430]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:55:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:55:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:16.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:16.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:55:16.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1010 B/s rd, 0 op/s 2024-05-15T15:55:16.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:16.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:16.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:55:16.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:55:16.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:16.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:16 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:17.735+0000 7ffabebee700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:55:18.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:18.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:55:18.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:18.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:18.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:18.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:18.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:18.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:18.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:18.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:18.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:18.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:55:18.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:55:18.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:55:18.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:55:18.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:55:18.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:18.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:55:18.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:18.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:18.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:18.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:18.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:18.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:18.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:18.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:18.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:18.365 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:18 smithi043 ceph-mon[24430]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:18.365 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:18 smithi043 ceph-mon[29127]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:18.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:18 smithi053 ceph-mon[67220]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:18.838 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:55:19.475 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:19 smithi043 ceph-mon[24430]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:19.475 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:19.475 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:19.475 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:19.475 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:19.475 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.475 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.475 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.476 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:55:19.477 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:55:19.478 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:55:19.478 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:55:19.478 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:19.478 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:19.478 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:19.478 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:19.478 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:19.478 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.478 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.478 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.479 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:19 smithi043 ceph-mon[24430]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:55:19.480 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:19 smithi043 ceph-mon[24430]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:19.480 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:19 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:19.480 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:19.480 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:19.480 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:19.480 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:19.480 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.480 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.480 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:19.481 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:55:19.482 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:19.483 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:19.484 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:19.484 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:19.484 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:19.484 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:19.484 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:19.484 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.484 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.484 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:19 smithi043 ceph-mon[29127]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:55:19.484 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:19 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:19.581 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:55:19.581 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (2m) 2m ago 24m 15.9M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:55:19.581 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (113s) 39s ago 24m 74.6M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:55:19.581 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 2m ago 23m 60.2M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:55:19.581 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (3m) 39s ago 29m 456M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:55:19.581 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (13m) 2m ago 30m 506M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (30m) 2m ago 30m 79.2M 2048M 17.2.0 e1d6a67b021e 784dab08799b 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (43s) 39s ago 29m 20.0M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (29m) 2m ago 29m 60.3M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (3m) 2m ago 25m 8334k - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (3m) 39s ago 25m 16.8M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (28m) 2m ago 28m 53.8M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (28m) 2m ago 28m 58.0M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (27m) 2m ago 27m 50.4M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (27m) 2m ago 27m 53.8M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (27m) 39s ago 27m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (26m) 39s ago 26m 51.6M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (26m) 39s ago 26m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:55:19.582 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (25m) 39s ago 25m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:55:19.583 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (2m) 39s ago 25m 66.6M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:55:19.583 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (23m) 2m ago 23m 76.2M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:55:19.583 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (23m) 39s ago 23m 75.6M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:55:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:19 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:55:19.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:19.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:19.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:19.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:19.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:19.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:19.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:19.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:19 smithi053 ceph-mon[67220]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:55:19.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:19 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:20.004 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:55:20.004 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:55:20.004 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 1 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: "mds": {}, 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:55:20.005 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-05-15T15:55:20.006 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T15:55:20.006 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:55:20.006 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:55:20.356 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:55:20.356 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", 2024-05-15T15:55:20.356 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:55:20.356 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi043", 2024-05-15T15:55:20.356 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:55:20.356 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "", 2024-05-15T15:55:20.356 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 image", 2024-05-15T15:55:20.356 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:55:20.357 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:55:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:20 smithi043 ceph-mon[24430]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:20 smithi043 ceph-mon[24430]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:20 smithi043 ceph-mon[24430]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 629 B/s rd, 0 op/s 2024-05-15T15:55:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:20 smithi043 ceph-mon[24430]: from='client.15648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:20.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:20 smithi043 ceph-mon[24430]: from='client.? 172.21.15.43:0/287717131' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:20.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:20 smithi043 ceph-mon[29127]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:20.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:20 smithi043 ceph-mon[29127]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:20.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:20 smithi043 ceph-mon[29127]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 629 B/s rd, 0 op/s 2024-05-15T15:55:20.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:20 smithi043 ceph-mon[29127]: from='client.15648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:20.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:20 smithi043 ceph-mon[29127]: from='client.? 172.21.15.43:0/287717131' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:20.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:20 smithi053 ceph-mon[67220]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:20.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:20 smithi053 ceph-mon[67220]: from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:20.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:20 smithi053 ceph-mon[67220]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 629 B/s rd, 0 op/s 2024-05-15T15:55:20.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:20 smithi053 ceph-mon[67220]: from='client.15648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:20.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:20 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/287717131' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: from='client.15660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:55:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: Upgrade: Setting container_image for all mgr 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[29127]: Upgrade: It appears safe to stop mon.a 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: from='client.15660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:55:21.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:55:21.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:21.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:21.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: Upgrade: Setting container_image for all mgr 2024-05-15T15:55:21.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:21.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-15T15:55:21.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-15T15:55:21.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:21 smithi043 ceph-mon[24430]: Upgrade: It appears safe to stop mon.a 2024-05-15T15:55:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: from='client.15660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:55:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:55:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mgr 2024-05-15T15:55:21.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:21.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-15T15:55:21.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-05-15T15:55:21.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:21 smithi053 ceph-mon[67220]: Upgrade: It appears safe to stop mon.a 2024-05-15T15:55:22.823 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:22 smithi043 ceph-mon[24430]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:22.823 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:22 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:22.823 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:22 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:55:22.823 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:22 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:55:22.823 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:22 smithi043 ceph-mon[24430]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:22.823 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:22 smithi043 ceph-mon[29127]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:22.824 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:22 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:22.824 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:22 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:55:22.824 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:22 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:55:22.824 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:22 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:22 smithi053 ceph-mon[67220]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:22 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' 2024-05-15T15:55:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:22 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:55:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:22 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:55:22.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:22 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:23.811 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:23 smithi043 ceph-mon[24430]: Upgrade: Updating mon.a 2024-05-15T15:55:23.811 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:23 smithi043 ceph-mon[24430]: Deploying daemon mon.a on smithi043 2024-05-15T15:55:23.812 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:23 smithi043 ceph-mon[29127]: Upgrade: Updating mon.a 2024-05-15T15:55:23.812 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:23 smithi043 ceph-mon[29127]: Deploying daemon mon.a on smithi043 2024-05-15T15:55:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:23 smithi053 ceph-mon[67220]: Upgrade: Updating mon.a 2024-05-15T15:55:23.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:23 smithi053 ceph-mon[67220]: Deploying daemon mon.a on smithi043 2024-05-15T15:55:24.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:23 smithi043 systemd[1]: Stopping Ceph mon.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:55:24.466 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a[24426]: 2024-05-15T15:55:24.121+0000 7f94e62cd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T15:55:24.466 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a[24426]: 2024-05-15T15:55:24.121+0000 7f94e62cd700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-05-15T15:55:24.466 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:24 smithi043 podman[82360]: 2024-05-15 15:55:24.183649981 +0000 UTC m=+0.183749200 container died 784dab08799bb027c2a62bbc7910f26443488e270e633f6ab6e404402dc29cf1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, release=754, build-date=2022-05-03T08:36:31.336870) 2024-05-15T15:55:24.466 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:55:24] ENGINE Bus STOPPING 2024-05-15T15:55:24.466 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:55:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-05-15T15:55:24.467 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:55:24] ENGINE Bus STOPPED 2024-05-15T15:55:24.467 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:55:24] ENGINE Bus STARTING 2024-05-15T15:55:24.467 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:55:24] ENGINE Serving on http://:::9283 2024-05-15T15:55:24.467 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:24 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:55:24] ENGINE Bus STARTED 2024-05-15T15:55:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:24 smithi043 podman[82360]: 2024-05-15 15:55:24.588726589 +0000 UTC m=+0.588825805 container cleanup 784dab08799bb027c2a62bbc7910f26443488e270e633f6ab6e404402dc29cf1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, version=8, architecture=x86_64, maintainer=Guillaume Abrioux , distribution-scope=public, vendor=Red Hat, Inc., io.openshift.expose-services=, io.buildah.version=1.19.8, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, vcs-type=git) 2024-05-15T15:55:24.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:24 smithi043 bash[82360]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a 2024-05-15T15:55:25.238 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:24 smithi043 podman[82386]: 2024-05-15 15:55:24.873424349 +0000 UTC m=+0.684187995 container remove 784dab08799bb027c2a62bbc7910f26443488e270e633f6ab6e404402dc29cf1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-type=git, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, name=centos-stream, version=8) 2024-05-15T15:55:25.238 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.a.service: Deactivated successfully. 2024-05-15T15:55:25.238 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 systemd[1]: Stopped Ceph mon.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:55:25.238 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.a.service: Consumed 28.340s CPU time. 2024-05-15T15:55:25.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 systemd[1]: Starting Ceph mon.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:55:25.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 podman[82513]: 2024-05-15 15:55:25.359048369 +0000 UTC m=+0.018461439 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:55:25.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 podman[82513]: 2024-05-15 15:55:25.475235058 +0000 UTC m=+0.134648134 container create c488ab1aa87e8875e30f777ad83cba6ef4127aca8d26c407ac07409bec666cb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-05-15T15:55:26.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 podman[82513]: 2024-05-15 15:55:25.729256025 +0000 UTC m=+0.388669081 container init c488ab1aa87e8875e30f777ad83cba6ef4127aca8d26c407ac07409bec666cb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, maintainer=Guillaume Abrioux ) 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 podman[82513]: 2024-05-15 15:55:25.733124305 +0000 UTC m=+0.392537369 container start c488ab1aa87e8875e30f777ad83cba6ef4127aca8d26c407ac07409bec666cb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: set uid:gid to 167:167 (ceph:ceph) 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev), process ceph-mon, pid 2 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: pidfile_write: ignore empty --pid-file 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: load: jerasure load: lrc 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: RocksDB version: 7.9.2 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Git sha 0 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Compile date 2024-05-15 11:18:19 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: DB SUMMARY 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: DB Session ID: 35MXY9W563GN0OYDQ3MW 2024-05-15T15:55:26.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: CURRENT file: CURRENT 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: IDENTITY file: IDENTITY 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: MANIFEST file: MANIFEST-000015 size: 3080 Bytes 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000063.sst 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000061.log size: 675576 ; 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.error_if_exists: 0 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.create_if_missing: 0 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.paranoid_checks: 1 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.env: 0x564637a38c60 2024-05-15T15:55:26.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.fs: PosixFileSystem 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.info_log: 0x564638d25ac0 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_file_opening_threads: 16 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.statistics: (nil) 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.use_fsync: 0 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_log_file_size: 0 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.keep_log_file_num: 1000 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.recycle_log_file_num: 0 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.allow_fallocate: 1 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.allow_mmap_reads: 0 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.allow_mmap_writes: 0 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.use_direct_reads: 0 2024-05-15T15:55:26.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.create_missing_column_families: 0 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.db_log_dir: 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.wal_dir: 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.advise_random_on_open: 1 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.db_write_buffer_size: 0 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.write_buffer_manager: 0x564638cf81e0 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-15T15:55:26.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.rate_limiter: (nil) 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.wal_recovery_mode: 2 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.enable_thread_tracking: 0 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.enable_pipelined_write: 0 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.unordered_write: 0 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.row_cache: None 2024-05-15T15:55:26.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.wal_filter: None 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.allow_ingest_behind: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.two_write_queues: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.manual_wal_flush: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.wal_compression: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.atomic_flush: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.log_readahead_size: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.best_efforts_recovery: 0 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-15T15:55:26.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.allow_data_in_errors: 0 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.db_host_id: __hostname__ 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_background_jobs: 2 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_background_compactions: -1 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_subcompactions: 1 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_total_wal_size: 0 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-15T15:55:26.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_open_files: -1 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bytes_per_sync: 0 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_readahead_size: 0 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_background_flushes: -1 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Compression algorithms supported: 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: kZSTD supported: 0 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: kXpressCompression supported: 0 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: kLZ4HCCompression supported: 1 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: kZlibCompression supported: 1 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: kSnappyCompression supported: 1 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: kLZ4Compression supported: 1 2024-05-15T15:55:26.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: kBZip2Compression supported: 0 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.merge_operator: 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_filter: None 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_filter_factory: None 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.sst_partitioner_factory: None 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564638d25bc0) 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout: cache_index_and_filter_blocks: 1 2024-05-15T15:55:26.096 INFO:journalctl@ceph.mon.a.smithi043.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: pin_top_level_index_and_filter: 1 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: index_type: 0 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: data_block_index_type: 0 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: index_shortening: 1 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: checksum: 4 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: no_block_cache: 0 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_cache: 0x564638d29090 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_cache_name: BinnedLRUCache 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_cache_options: 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: capacity : 536870912 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: num_shard_bits : 4 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: strict_capacity_limit : 0 2024-05-15T15:55:26.097 INFO:journalctl@ceph.mon.a.smithi043.stdout: high_pri_pool_ratio: 0.000 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_cache_compressed: (nil) 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: persistent_cache: (nil) 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_size: 4096 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_size_deviation: 10 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_restart_interval: 16 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: index_block_restart_interval: 1 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: metadata_block_size: 4096 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: partition_filters: 0 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_delta_encoding: 1 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: filter_policy: bloomfilter 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: whole_key_filtering: 1 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: verify_compression: 0 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: read_amp_bytes_per_bit: 0 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: format_version: 5 2024-05-15T15:55:26.098 INFO:journalctl@ceph.mon.a.smithi043.stdout: enable_index_compression: 1 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: block_align: 0 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: max_auto_readahead_size: 262144 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: prepopulate_block_cache: 0 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: initial_auto_readahead_size: 8192 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout: num_file_reads_for_auto_readahead: 2 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.write_buffer_size: 33554432 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_write_buffer_number: 2 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compression: NoCompression 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bottommost_compression: Disabled 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.prefix_extractor: nullptr 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.num_levels: 7 2024-05-15T15:55:26.099 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compression_opts.level: 32767 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compression_opts.strategy: 0 2024-05-15T15:55:26.100 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compression_opts.enabled: false 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.target_file_size_base: 67108864 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-15T15:55:26.101 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.arena_block_size: 1048576 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.disable_auto_compactions: 0 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-15T15:55:26.102 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.table_properties_collectors: 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.inplace_update_support: 0 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-15T15:55:26.103 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.bloom_locality: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.max_successive_merges: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.paranoid_file_checks: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.force_consistency_checks: 1 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.report_bg_io_stats: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.ttl: 2592000 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.enable_blob_files: false 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.min_blob_size: 0 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.blob_file_size: 268435456 2024-05-15T15:55:26.104 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.blob_file_starting_level: 0 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 63.sst 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 65, last_sequence is 29198, log_number is 61,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 61 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b5f0a75b-4a50-4018-9a0c-fda0a952ab24 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715788525764758, "job": 1, "event": "recovery_started", "wal_files": [61]} 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #61 mode 2 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715788525769785, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 66, "file_size": 663630, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 29131, "largest_seqno": 29711, "table_properties": {"data_size": 660236, "index_size": 1612, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 773, "raw_key_size": 7260, "raw_average_key_size": 24, "raw_value_size": 654257, "raw_average_value_size": 2202, "num_data_blocks": 71, "num_entries": 297, "num_filter_entries": 297, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1715788525, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b5f0a75b-4a50-4018-9a0c-fda0a952ab24", "db_session_id": "35MXY9W563GN0OYDQ3MW", "orig_file_number": 66, "seqno_to_time_mapping": "N/A"}} 2024-05-15T15:55:26.105 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715788525769915, "job": 1, "event": "recovery_finished"} 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/version_set.cc:5047] Creating manifest 68 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000061.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564638e1a000 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: DB pointer 0x564638e04000 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** DB Stats ** 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:55:26.106 INFO:journalctl@ceph.mon.a.smithi043.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: L0 1/0 648.08 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 216.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: L6 1/0 9.81 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Sum 2/0 10.45 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 216.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 216.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:55:26.107 INFO:journalctl@ceph.mon.a.smithi043.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 216.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Cumulative compaction: 0.00 GB write, 78.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Interval compaction: 0.00 GB write, 78.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:55:26.108 INFO:journalctl@ceph.mon.a.smithi043.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: Block cache BinnedLRUCache@0x564638d29090#2 capacity: 512.00 MB usage: 2.73 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.91 KB,0.000172853%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: starting mon.a rank 0 at public addrs [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] at bind addrs [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: mon.a@-1(???) e3 preinit fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: mon.a@-1(???).mds e1 new map 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: mon.a@-1(???).mds e1 print_map 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: e1 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: legacy client fscid: -1 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout: No filesystems configured 2024-05-15T15:55:26.109 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: mon.a@-1(???).osd e106 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-15T15:55:26.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: mon.a@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-05-15T15:55:26.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: mon.a@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-05-15T15:55:26.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: mon.a@-1(???).osd e106 crush map has features 432629239337189376, adjusting msgr requires 2024-05-15T15:55:26.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 ceph-mon[82531]: mon.a@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-05-15T15:55:26.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 bash[82513]: c488ab1aa87e8875e30f777ad83cba6ef4127aca8d26c407ac07409bec666cb4 2024-05-15T15:55:26.110 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:25 smithi043 systemd[1]: Started Ceph mon.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:55:26.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:55:26] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-05-15T15:55:27.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:27 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ignoring --setuser ceph since I am not root 2024-05-15T15:55:27.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:27 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ignoring --setgroup ceph since I am not root 2024-05-15T15:55:27.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mgr[62161]: -- 172.21.15.43:0/3277461915 <== mon.1 v2:172.21.15.43:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564dac3405a0 con 0x564dac314800 2024-05-15T15:55:27.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:27 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:27.397+0000 7fbfcf25d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: mon.a calling monitor election 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: monmap epoch 3 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: last_changed 2024-05-15T15:26:03.818101+0000 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: created 2024-05-15T15:24:25.819270+0000 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: min_mon_release 17 (quincy) 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: election_strategy: 1 2024-05-15T15:55:27.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: 0: [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] mon.a 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: 1: [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] mon.c 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: 2: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.b 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: fsmap 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: osdmap e106: 8 total, 8 up, 8 in 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: mgrmap e42: y(active, since 4m), standbys: x 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: mgrmap e43: y(active, since 4m), standbys: x 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[29127]: from='mgr.25186 ' entity='' 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:55:27.588 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: mon.a calling monitor election 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: monmap epoch 3 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: last_changed 2024-05-15T15:26:03.818101+0000 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: created 2024-05-15T15:24:25.819270+0000 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: min_mon_release 17 (quincy) 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: election_strategy: 1 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: 0: [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] mon.a 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: 1: [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] mon.c 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: 2: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.b 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: fsmap 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: osdmap e106: 8 total, 8 up, 8 in 2024-05-15T15:55:27.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: mgrmap e42: y(active, since 4m), standbys: x 2024-05-15T15:55:27.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:55:27.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:55:27.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:55:27.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: mgrmap e43: y(active, since 4m), standbys: x 2024-05-15T15:55:27.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:27 smithi043 ceph-mon[82531]: from='mgr.25186 ' entity='' 2024-05-15T15:55:27.596 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: ignoring --setuser ceph since I am not root 2024-05-15T15:55:27.596 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: ignoring --setgroup ceph since I am not root 2024-05-15T15:55:27.596 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mgr[60211]: -- 172.21.15.53:0/3642036744 <== mon.2 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5642856bc5a0 con 0x564285691000 2024-05-15T15:55:27.596 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:27.401+0000 7fbb88b5f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-15T15:55:27.596 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:27.594+0000 7fbb88b5f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: from='mgr.25186 172.21.15.43:0/3018462057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: mon.a calling monitor election 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: monmap epoch 3 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: last_changed 2024-05-15T15:26:03.818101+0000 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: created 2024-05-15T15:24:25.819270+0000 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: min_mon_release 17 (quincy) 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: election_strategy: 1 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: 0: [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] mon.a 2024-05-15T15:55:27.597 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: 1: [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] mon.c 2024-05-15T15:55:27.598 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: 2: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.b 2024-05-15T15:55:27.598 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: fsmap 2024-05-15T15:55:27.598 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: osdmap e106: 8 total, 8 up, 8 in 2024-05-15T15:55:27.598 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: mgrmap e42: y(active, since 4m), standbys: x 2024-05-15T15:55:27.598 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:55:27.598 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:55:27.598 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:55:27.598 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: mgrmap e43: y(active, since 4m), standbys: x 2024-05-15T15:55:27.598 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:27 smithi053 ceph-mon[67220]: from='mgr.25186 ' entity='' 2024-05-15T15:55:27.845 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:27 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:27.591+0000 7fbfcf25d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:55:27.845 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:27 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:27.699+0000 7fbfcf25d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:55:27.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:27.701+0000 7fbb88b5f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-15T15:55:28.124 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:28.031+0000 7fbfcf25d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:55:28.124 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:28.122+0000 7fbfcf25d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:55:28.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:28.031+0000 7fbb88b5f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-15T15:55:28.380 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:28.121+0000 7fbb88b5f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-15T15:55:29.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:29 smithi043 ceph-mon[82531]: mgrmap e44: y(active, since 4m), standbys: x 2024-05-15T15:55:29.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:29 smithi043 ceph-mon[29127]: mgrmap e44: y(active, since 4m), standbys: x 2024-05-15T15:55:29.563 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:29 smithi053 ceph-mon[67220]: mgrmap e44: y(active, since 4m), standbys: x 2024-05-15T15:55:29.563 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:29.381+0000 7fbb88b5f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:55:29.563 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:29.561+0000 7fbb88b5f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:55:29.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:29 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:29.395+0000 7fbfcf25d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-15T15:55:29.736 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:29 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:29.575+0000 7fbfcf25d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-15T15:55:29.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:29.741+0000 7fbb88b5f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:55:30.073 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:29 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:29.761+0000 7fbfcf25d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-15T15:55:30.073 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:30.070+0000 7fbfcf25d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:55:30.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:30.169+0000 7fbfcf25d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:55:30.361 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:30.047+0000 7fbb88b5f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-15T15:55:30.361 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:30.144+0000 7fbb88b5f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-15T15:55:30.361 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:30.358+0000 7fbb88b5f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:55:30.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:30.563+0000 7fbb88b5f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:55:30.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:30.385+0000 7fbfcf25d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-15T15:55:30.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:30.592+0000 7fbfcf25d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-15T15:55:31.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:31.236+0000 7fbfcf25d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:55:31.630 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:31.205+0000 7fbb88b5f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-15T15:55:31.976 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:31.654+0000 7fbb88b5f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:55:31.977 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:31.757+0000 7fbb88b5f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:55:31.977 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:31.974+0000 7fbb88b5f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-15T15:55:32.010 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:31.687+0000 7fbfcf25d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-15T15:55:32.010 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:31.791+0000 7fbfcf25d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-15T15:55:32.010 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:32.007+0000 7fbfcf25d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-15T15:55:32.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:32.579+0000 7fbfcf25d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:55:32.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:32.679+0000 7fbfcf25d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:55:32.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:32 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:32.548+0000 7fbb88b5f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-15T15:55:32.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:32 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:32.647+0000 7fbb88b5f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-15T15:55:33.205 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:32.944+0000 7fbfcf25d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:55:33.205 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:33.088+0000 7fbfcf25d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:55:33.205 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:33.202+0000 7fbfcf25d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:55:33.282 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:32 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:32.908+0000 7fbb88b5f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-15T15:55:33.283 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:33.051+0000 7fbb88b5f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-15T15:55:33.283 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:33.153+0000 7fbb88b5f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-15T15:55:33.283 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:33.280+0000 7fbb88b5f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:55:33.519 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:33.329+0000 7fbfcf25d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-15T15:55:33.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:33.650+0000 7fbfcf25d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:55:33.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:55:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:55:33.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T15:55:33.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:55:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:55:33.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T15:55:33.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:33.598+0000 7fbb88b5f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-15T15:55:34.545 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:34.243+0000 7fbb88b5f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:55:34.545 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:34.479+0000 7fbb88b5f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:55:34.545 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: [15/May/2024:15:55:34] ENGINE Bus STARTING 2024-05-15T15:55:34.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:34.298+0000 7fbfcf25d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-15T15:55:34.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:34.534+0000 7fbfcf25d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-15T15:55:34.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[82531]: Standby manager daemon x restarted 2024-05-15T15:55:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[82531]: Standby manager daemon x started 2024-05-15T15:55:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[82531]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:55:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[82531]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:55:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[82531]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:55:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[82531]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:55:34.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[29127]: Standby manager daemon x restarted 2024-05-15T15:55:34.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[29127]: Standby manager daemon x started 2024-05-15T15:55:34.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:55:34.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:55:34.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:55:34.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:34 smithi043 ceph-mon[29127]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:55:34.880 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: CherryPy Checker: 2024-05-15T15:55:34.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: The Application mounted at '' has an empty config. 2024-05-15T15:55:34.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: 2024-05-15T15:55:34.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: [15/May/2024:15:55:34] ENGINE Serving on http://:::9283 2024-05-15T15:55:34.881 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 15:55:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x[60207]: [15/May/2024:15:55:34] ENGINE Bus STARTED 2024-05-15T15:55:34.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:34 smithi053 ceph-mon[67220]: Standby manager daemon x restarted 2024-05-15T15:55:34.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:34 smithi053 ceph-mon[67220]: Standby manager daemon x started 2024-05-15T15:55:34.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:34 smithi053 ceph-mon[67220]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-05-15T15:55:34.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:34 smithi053 ceph-mon[67220]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-15T15:55:34.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:34 smithi053 ceph-mon[67220]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-05-15T15:55:34.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:34 smithi053 ceph-mon[67220]: from='mgr.? 172.21.15.53:0/706534756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-15T15:55:35.086 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:55:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:55:34.868Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=3 2024-05-15T15:55:35.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:55:34] ENGINE Bus STARTING 2024-05-15T15:55:35.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CherryPy Checker: 2024-05-15T15:55:35.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: The Application mounted at '' has an empty config. 2024-05-15T15:55:35.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:35.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:55:34] ENGINE Serving on http://:::9283 2024-05-15T15:55:35.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: [15/May/2024:15:55:34] ENGINE Bus STARTED 2024-05-15T15:55:35.792 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T15:55:35.421Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=3 2024-05-15T15:55:35.793 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:35.561+0000 7fbf8c6f9700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-05-15T15:55:35.793 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: mgrmap e45: y(active, since 4m), standbys: x 2024-05-15T15:55:35.793 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: Active manager daemon y restarted 2024-05-15T15:55:35.793 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: Activating manager daemon y 2024-05-15T15:55:35.793 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: osdmap e107: 8 total, 8 up, 8 in 2024-05-15T15:55:35.793 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: mgrmap e46: y(active, starting, since 0.0150999s), standbys: x 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:55:35.794 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: Manager daemon y is now available 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: mgrmap e45: y(active, since 4m), standbys: x 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: Active manager daemon y restarted 2024-05-15T15:55:35.795 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: Activating manager daemon y 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: osdmap e107: 8 total, 8 up, 8 in 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: mgrmap e46: y(active, starting, since 0.0150999s), standbys: x 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:55:35.796 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:55:35.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:55:35.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:55:35.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: Manager daemon y is now available 2024-05-15T15:55:35.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:35.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:55:35.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:55:35.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:55:35.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:55:35.797 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:35 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:55:35.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: mgrmap e45: y(active, since 4m), standbys: x 2024-05-15T15:55:35.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: Active manager daemon y restarted 2024-05-15T15:55:35.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: Activating manager daemon y 2024-05-15T15:55:35.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: osdmap e107: 8 total, 8 up, 8 in 2024-05-15T15:55:35.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: mgrmap e46: y(active, starting, since 0.0150999s), standbys: x 2024-05-15T15:55:35.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:55:35.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:55:35.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:55:35.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-15T15:55:35.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-15T15:55:35.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:55:35.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: Manager daemon y is now available 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:55:35.883 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-05-15T15:55:35.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:55:35.884 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-05-15T15:55:36.733 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:55:36] "GET /metrics HTTP/1.1" 200 35191 "" "Prometheus/2.43.0" 2024-05-15T15:55:36.734 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:36 smithi043 ceph-mon[29127]: mgrmap e47: y(active, since 1.01737s), standbys: x 2024-05-15T15:55:36.734 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:36 smithi043 ceph-mon[29127]: [15/May/2024:15:55:35] ENGINE Bus STARTING 2024-05-15T15:55:36.734 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:36 smithi043 ceph-mon[29127]: [15/May/2024:15:55:35] ENGINE Serving on http://172.21.15.43:8765 2024-05-15T15:55:36.734 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:36 smithi043 ceph-mon[29127]: [15/May/2024:15:55:35] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:55:36.734 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:36 smithi043 ceph-mon[29127]: [15/May/2024:15:55:35] ENGINE Bus STARTED 2024-05-15T15:55:36.734 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:36 smithi043 ceph-mon[82531]: mgrmap e47: y(active, since 1.01737s), standbys: x 2024-05-15T15:55:36.734 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:36 smithi043 ceph-mon[82531]: [15/May/2024:15:55:35] ENGINE Bus STARTING 2024-05-15T15:55:36.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:36 smithi043 ceph-mon[82531]: [15/May/2024:15:55:35] ENGINE Serving on http://172.21.15.43:8765 2024-05-15T15:55:36.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:36 smithi043 ceph-mon[82531]: [15/May/2024:15:55:35] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:55:36.735 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:36 smithi043 ceph-mon[82531]: [15/May/2024:15:55:35] ENGINE Bus STARTED 2024-05-15T15:55:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:36 smithi053 ceph-mon[67220]: mgrmap e47: y(active, since 1.01737s), standbys: x 2024-05-15T15:55:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:36 smithi053 ceph-mon[67220]: [15/May/2024:15:55:35] ENGINE Bus STARTING 2024-05-15T15:55:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:36 smithi053 ceph-mon[67220]: [15/May/2024:15:55:35] ENGINE Serving on http://172.21.15.43:8765 2024-05-15T15:55:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:36 smithi053 ceph-mon[67220]: [15/May/2024:15:55:35] ENGINE Serving on https://172.21.15.43:7150 2024-05-15T15:55:36.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:36 smithi053 ceph-mon[67220]: [15/May/2024:15:55:35] ENGINE Bus STARTED 2024-05-15T15:55:37.756 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:37 smithi043 ceph-mon[82531]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-15T15:55:37.756 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:37 smithi043 ceph-mon[82531]: Cluster is now healthy 2024-05-15T15:55:37.756 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:37 smithi043 ceph-mon[82531]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:55:37.756 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:37 smithi043 ceph-mon[82531]: mgrmap e48: y(active, since 2s), standbys: x 2024-05-15T15:55:37.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:37 smithi043 ceph-mon[29127]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-15T15:55:37.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:37 smithi043 ceph-mon[29127]: Cluster is now healthy 2024-05-15T15:55:37.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:37 smithi043 ceph-mon[29127]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:55:37.757 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:37 smithi043 ceph-mon[29127]: mgrmap e48: y(active, since 2s), standbys: x 2024-05-15T15:55:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:37 smithi053 ceph-mon[67220]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-15T15:55:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:37 smithi053 ceph-mon[67220]: Cluster is now healthy 2024-05-15T15:55:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:37 smithi053 ceph-mon[67220]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:55:37.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:37 smithi053 ceph-mon[67220]: mgrmap e48: y(active, since 2s), standbys: x 2024-05-15T15:55:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[82531]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:55:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[82531]: mgrmap e49: y(active, since 4s), standbys: x 2024-05-15T15:55:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[29127]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:55:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[29127]: mgrmap e49: y(active, since 4s), standbys: x 2024-05-15T15:55:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:39 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:39 smithi053 ceph-mon[67220]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:55:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:39 smithi053 ceph-mon[67220]: mgrmap e49: y(active, since 4s), standbys: x 2024-05-15T15:55:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:39 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:39 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:39 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:39.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:39 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:41 smithi053 ceph-mon[67220]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:55:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:41 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:41 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:41 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:41.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:41 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:42.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:41 smithi043 ceph-mon[29127]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:55:42.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:41 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:42.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:41 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:42.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:41 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:42.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:41 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:42.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:41 smithi043 ceph-mon[82531]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:55:42.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:41 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:42.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:41 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:42.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:41 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:42.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:41 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:43.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:43 smithi053 ceph-mon[67220]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:55:43.899 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:43 smithi043 ceph-mon[82531]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:55:43.899 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:43 smithi043 ceph-mon[29127]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-05-15T15:55:45.580 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.580 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.580 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.580 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.580 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:45.580 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:55:45.580 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.581 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.581 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.581 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.581 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:45.581 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:45 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:55:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:45 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:45 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:45 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:45 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:55:46.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: Detected new or changed devices on smithi043 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.533 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:55:46.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:55:46.534 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:46 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:46.540 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:55:46] "GET /metrics HTTP/1.1" 200 35191 "" "Prometheus/2.43.0" 2024-05-15T15:55:46.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: Detected new or changed devices on smithi043 2024-05-15T15:55:46.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:55:46.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:55:46.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-15T15:55:46.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:55:46.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: Detected new or changed devices on smithi043 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.conf 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.conf 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.conf 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: Updating smithi043:/etc/ceph/ceph.client.admin.keyring 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: Updating smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: Updating smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/config/ceph.client.admin.keyring 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.542 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:46.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:55:46.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:55:46.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:46 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:47.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:47 smithi043 ceph-mon[82531]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:55:47.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:47 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:55:47.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:47 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:55:47.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:47 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:47.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:47 smithi043 ceph-mon[82531]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T15:55:47.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:47 smithi043 ceph-mon[29127]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:55:47.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:47 smithi043 ceph-mon[29127]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:55:47.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:47 smithi043 ceph-mon[29127]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:55:47.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:47 smithi043 ceph-mon[29127]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:47.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:47 smithi043 ceph-mon[29127]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T15:55:47.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:47.342+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:47.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:55:47.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:47.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:55:47.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:47.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:47.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:47 smithi053 ceph-mon[67220]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:55:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:47 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:55:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:47 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:55:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:47 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:47 smithi053 ceph-mon[67220]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T15:55:48.538 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:48 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:48.538 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:48 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:48.538 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:48 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-15T15:55:48.538 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:48 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-15T15:55:48.538 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:48 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:48.538 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:48 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:48.538 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:48 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-15T15:55:48.538 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:48 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-15T15:55:48.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:48 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:48.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:48 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:55:48.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:48 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-15T15:55:48.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:48 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-05-15T15:55:49.528 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[29127]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:49.528 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:49.529 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:55:49.530 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:49.531 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.532 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.532 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:49.532 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:49.532 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:49.532 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:49.532 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:55:49.532 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:49.532 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:49.532 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:49.532 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:49.533 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:55:49.534 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:49.535 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.537 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.537 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:55:49.537 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:49.538 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:49.538 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:49.538 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:55:49.538 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:49.538 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:49.538 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:49.538 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:49.538 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[29127]: Upgrade: It appears safe to stop mon.c 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[29127]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[29127]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:49.539 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:49.540 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:49.540 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:49.540 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.540 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[82531]: Upgrade: It appears safe to stop mon.c 2024-05-15T15:55:49.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[82531]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:55:49.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:49.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:55:49.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:55:49.540 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:49 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:49 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:49.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:55:49.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:55:49.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:55:49.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:55:49.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:55:49.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:55:49.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:55:49.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:55:49.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:49 smithi053 ceph-mon[67220]: Upgrade: It appears safe to stop mon.c 2024-05-15T15:55:49.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:49 smithi053 ceph-mon[67220]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:55:49.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:49 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:49.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:49 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:55:49.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:49 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:55:49.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:49 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:55:50.481 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:50 smithi043 ceph-mon[29127]: Upgrade: Updating mon.c 2024-05-15T15:55:50.481 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:50 smithi043 ceph-mon[29127]: Deploying daemon mon.c on smithi043 2024-05-15T15:55:50.481 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:50 smithi043 ceph-mon[29127]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:55:50.481 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:50 smithi043 ceph-mon[82531]: Upgrade: Updating mon.c 2024-05-15T15:55:50.481 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:50 smithi043 ceph-mon[82531]: Deploying daemon mon.c on smithi043 2024-05-15T15:55:50.482 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:50 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:55:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:50 smithi053 ceph-mon[67220]: Upgrade: Updating mon.c 2024-05-15T15:55:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:50 smithi053 ceph-mon[67220]: Deploying daemon mon.c on smithi043 2024-05-15T15:55:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:50 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:55:50.750 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:55:51.061 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:50 smithi043 systemd[1]: Stopping Ceph mon.c for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:55:51.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c[29123]: 2024-05-15T15:55:51.058+0000 7faf6bb46700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T15:55:51.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c[29123]: 2024-05-15T15:55:51.058+0000 7faf6bb46700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-05-15T15:55:51.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:51 smithi043 podman[87866]: 2024-05-15 15:55:51.089938286 +0000 UTC m=+0.155007375 container died 44c487fc3e4119314217a291c2701b77ad9418806db84eec9194faf234830d20 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, distribution-scope=public, RELEASE=HEAD, io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, ceph=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, version=8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD) 2024-05-15T15:55:51.472 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:55:51.472 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (2m) 12s ago 24m 19.1M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:55:51.472 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (2m) 12s ago 24m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:55:51.472 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 12s ago 24m 64.6M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:55:51.472 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (4m) 12s ago 29m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (14m) 12s ago 31m 479M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (26s) 12s ago 31m 29.9M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (75s) 12s ago 29m 38.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (29m) 12s ago 29m 64.0M 2048M 17.2.0 e1d6a67b021e 44c487fc3e41 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (3m) 12s ago 25m 8606k - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (3m) 12s ago 25m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (29m) 12s ago 29m 53.9M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (28m) 12s ago 28m 58.1M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (28m) 12s ago 28m 50.5M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (28m) 12s ago 28m 53.5M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (27m) 12s ago 27m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (27m) 12s ago 27m 51.9M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (26m) 12s ago 26m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (26m) 12s ago 26m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:55:51.473 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (3m) 12s ago 25m 68.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:55:51.474 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (24m) 12s ago 24m 77.4M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:55:51.474 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (24m) 12s ago 24m 76.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:55:51.745 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:51 smithi043 podman[87866]: 2024-05-15 15:55:51.450040924 +0000 UTC m=+0.515110004 container cleanup 44c487fc3e4119314217a291c2701b77ad9418806db84eec9194faf234830d20 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, name=centos-stream, io.buildah.version=1.19.8, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8) 2024-05-15T15:55:51.745 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:51 smithi043 bash[87866]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:55:51.917 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:55:51.918 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:55:51.918 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:55:51.918 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:55:51.918 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-05-15T15:55:51.918 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 4 2024-05-15T15:55:51.918 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:55:51.918 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:55:52.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:51 smithi043 podman[87879]: 2024-05-15 15:55:51.743327226 +0000 UTC m=+0.654539970 container remove 44c487fc3e4119314217a291c2701b77ad9418806db84eec9194faf234830d20 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, release=754, GIT_BRANCH=HEAD, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , version=8, architecture=x86_64, name=centos-stream, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-15T15:55:52.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:51 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.c.service: Deactivated successfully. 2024-05-15T15:55:52.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:51 smithi043 systemd[1]: Stopped Ceph mon.c for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:55:52.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:51 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.c.service: Consumed 17.357s CPU time. 2024-05-15T15:55:52.300 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:55:52.301 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T15:55:52.301 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:55:52.301 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi043", 2024-05-15T15:55:52.301 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:55:52.301 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "1/2 daemons upgraded", 2024-05-15T15:55:52.301 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading mon daemons", 2024-05-15T15:55:52.301 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:55:52.301 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:55:52.496 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 systemd[1]: Starting Ceph mon.c for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:55:52.496 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 podman[88076]: 2024-05-15 15:55:52.264613933 +0000 UTC m=+0.018631131 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:55:52.496 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 podman[88076]: 2024-05-15 15:55:52.386226667 +0000 UTC m=+0.140243846 container create d4da75e1a85c1fc123e47a40e14a21313362b4d5ebc2efd866dd52daff2c749c (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T15:55:52.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 podman[88076]: 2024-05-15 15:55:52.630738762 +0000 UTC m=+0.384755951 container init d4da75e1a85c1fc123e47a40e14a21313362b4d5ebc2efd866dd52daff2c749c (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d) 2024-05-15T15:55:52.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 podman[88076]: 2024-05-15 15:55:52.63439971 +0000 UTC m=+0.388416900 container start d4da75e1a85c1fc123e47a40e14a21313362b4d5ebc2efd866dd52daff2c749c (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-05-15T15:55:52.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: set uid:gid to 167:167 (ceph:ceph) 2024-05-15T15:55:52.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev), process ceph-mon, pid 2 2024-05-15T15:55:52.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: pidfile_write: ignore empty --pid-file 2024-05-15T15:55:52.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: load: jerasure load: lrc 2024-05-15T15:55:52.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: RocksDB version: 7.9.2 2024-05-15T15:55:52.766 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Git sha 0 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Compile date 2024-05-15 11:18:19 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: DB SUMMARY 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: DB Session ID: E66RIIZLAG9ZWF7N37X0 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: CURRENT file: CURRENT 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: IDENTITY file: IDENTITY 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3195 Bytes 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 2794374 ; 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.error_if_exists: 0 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.create_if_missing: 0 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.paranoid_checks: 1 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-15T15:55:52.767 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-15T15:55:52.768 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.env: 0x55fb351dbc60 2024-05-15T15:55:52.768 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.fs: PosixFileSystem 2024-05-15T15:55:52.768 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.info_log: 0x55fb379e3ac0 2024-05-15T15:55:52.768 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_file_opening_threads: 16 2024-05-15T15:55:52.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.statistics: (nil) 2024-05-15T15:55:52.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.use_fsync: 0 2024-05-15T15:55:52.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_log_file_size: 0 2024-05-15T15:55:52.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-15T15:55:52.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-15T15:55:52.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.keep_log_file_num: 1000 2024-05-15T15:55:52.769 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.recycle_log_file_num: 0 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.allow_fallocate: 1 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.allow_mmap_reads: 0 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.allow_mmap_writes: 0 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.use_direct_reads: 0 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.create_missing_column_families: 0 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.db_log_dir: 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.wal_dir: 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-15T15:55:52.770 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.advise_random_on_open: 1 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.db_write_buffer_size: 0 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.write_buffer_manager: 0x55fb379b61e0 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.rate_limiter: (nil) 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.wal_recovery_mode: 2 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.enable_thread_tracking: 0 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.enable_pipelined_write: 0 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.unordered_write: 0 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-15T15:55:52.771 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.row_cache: None 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.wal_filter: None 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.allow_ingest_behind: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.two_write_queues: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.manual_wal_flush: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.wal_compression: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.atomic_flush: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.log_readahead_size: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.best_efforts_recovery: 0 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-15T15:55:52.772 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.allow_data_in_errors: 0 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.db_host_id: __hostname__ 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_background_jobs: 2 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_background_compactions: -1 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_subcompactions: 1 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_total_wal_size: 0 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_open_files: -1 2024-05-15T15:55:52.773 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bytes_per_sync: 0 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_readahead_size: 0 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_background_flushes: -1 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Compression algorithms supported: 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: kZSTD supported: 0 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: kXpressCompression supported: 0 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: kLZ4HCCompression supported: 1 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: kZlibCompression supported: 1 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: kSnappyCompression supported: 1 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: kLZ4Compression supported: 1 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: kBZip2Compression supported: 0 2024-05-15T15:55:52.774 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.merge_operator: 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_filter: None 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_filter_factory: None 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.sst_partitioner_factory: None 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fb379e3bc0) 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout: cache_index_and_filter_blocks: 1 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-15T15:55:52.775 INFO:journalctl@ceph.mon.c.smithi043.stdout: pin_top_level_index_and_filter: 1 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: index_type: 0 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: data_block_index_type: 0 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: index_shortening: 1 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: checksum: 4 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: no_block_cache: 0 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_cache: 0x55fb379e7090 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_cache_name: BinnedLRUCache 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_cache_options: 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: capacity : 536870912 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: num_shard_bits : 4 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: strict_capacity_limit : 0 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: high_pri_pool_ratio: 0.000 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_cache_compressed: (nil) 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: persistent_cache: (nil) 2024-05-15T15:55:52.776 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_size: 4096 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_size_deviation: 10 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_restart_interval: 16 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: index_block_restart_interval: 1 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: metadata_block_size: 4096 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: partition_filters: 0 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_delta_encoding: 1 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: filter_policy: bloomfilter 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: whole_key_filtering: 1 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: verify_compression: 0 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: read_amp_bytes_per_bit: 0 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: format_version: 5 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: enable_index_compression: 1 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: block_align: 0 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: max_auto_readahead_size: 262144 2024-05-15T15:55:52.777 INFO:journalctl@ceph.mon.c.smithi043.stdout: prepopulate_block_cache: 0 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout: initial_auto_readahead_size: 8192 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout: num_file_reads_for_auto_readahead: 2 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.write_buffer_size: 33554432 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_write_buffer_number: 2 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compression: NoCompression 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bottommost_compression: Disabled 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.prefix_extractor: nullptr 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.num_levels: 7 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-15T15:55:52.778 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compression_opts.level: 32767 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compression_opts.strategy: 0 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-15T15:55:52.779 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compression_opts.enabled: false 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.target_file_size_base: 67108864 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-15T15:55:52.780 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.arena_block_size: 1048576 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.disable_auto_compactions: 0 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-15T15:55:52.781 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.table_properties_collectors: 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.inplace_update_support: 0 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.bloom_locality: 0 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.max_successive_merges: 0 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.paranoid_file_checks: 0 2024-05-15T15:55:52.782 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.force_consistency_checks: 1 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.report_bg_io_stats: 0 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.ttl: 2592000 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.enable_blob_files: false 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.min_blob_size: 0 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.blob_file_size: 268435456 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.blob_file_starting_level: 0 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-15T15:55:52.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32678, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f76b58c3-f112-41c9-bd02-b9dd94378f6c 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715788552664816, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715788552672372, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1780008, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32684, "largest_seqno": 33139, "table_properties": {"data_size": 1776818, "index_size": 1600, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 5191, "raw_average_key_size": 24, "raw_value_size": 1771856, "raw_average_value_size": 8203, "num_data_blocks": 72, "num_entries": 216, "num_filter_entries": 216, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1715788552, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f76b58c3-f112-41c9-bd02-b9dd94378f6c", "db_session_id": "E66RIIZLAG9ZWF7N37X0", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715788552672484, "job": 1, "event": "recovery_finished"} 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55fb37ad8000 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: DB pointer 0x55fb37ac2000 2024-05-15T15:55:52.784 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** DB Stats ** 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: L0 1/0 1.70 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 511.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-15T15:55:52.785 INFO:journalctl@ceph.mon.c.smithi043.stdout: L6 1/0 10.85 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: Sum 2/0 12.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 511.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 511.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** Compaction Stats [default] ** 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 511.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-15T15:55:52.786 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: Cumulative compaction: 0.00 GB write, 170.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: Interval compaction: 0.00 GB write, 170.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: Block cache BinnedLRUCache@0x55fb379e7090#2 capacity: 512.00 MB usage: 2.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.66 KB,0.00012517%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: starting mon.c rank 1 at public addrs [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] at bind addrs [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: mon.c@-1(???) e3 preinit fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: mon.c@-1(???).mds e1 new map 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: mon.c@-1(???).mds e1 print_map 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: e1 2024-05-15T15:55:52.787 INFO:journalctl@ceph.mon.c.smithi043.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout: legacy client fscid: -1 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout: 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout: No filesystems configured 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: mon.c@-1(???).osd e107 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: mon.c@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: mon.c@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: mon.c@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 ceph-mon[88099]: mon.c@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 bash[88076]: d4da75e1a85c1fc123e47a40e14a21313362b4d5ebc2efd866dd52daff2c749c 2024-05-15T15:55:52.788 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:52 smithi043 systemd[1]: Started Ceph mon.c for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:55:54.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:54 smithi053 ceph-mon[67220]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:55:54.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:54 smithi043 ceph-mon[82531]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:55:54.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:54 smithi043 ceph-mon[88099]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-05-15T15:55:55.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: 2: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.b 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: fsmap 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: osdmap e107: 8 total, 8 up, 8 in 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: mgrmap e49: y(active, since 19s), standbys: x 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:55:55.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: mon.b calling monitor election 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: mon.c calling monitor election 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: mon.a calling monitor election 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: monmap epoch 4 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: last_changed 2024-05-15T15:55:54.230321+0000 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: created 2024-05-15T15:24:25.819270+0000 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: min_mon_release 19 (squid) 2024-05-15T15:55:55.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: election_strategy: 1 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: 0: [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] mon.a 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: 1: [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] mon.c 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: 2: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.b 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: fsmap 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: osdmap e107: 8 total, 8 up, 8 in 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: mgrmap e49: y(active, since 19s), standbys: x 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='' 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:55.339 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: 2: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.b 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: fsmap 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: osdmap e107: 8 total, 8 up, 8 in 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: mgrmap e49: y(active, since 19s), standbys: x 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:55:55.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: mon.b calling monitor election 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: mon.c calling monitor election 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: mon.a calling monitor election 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: monmap epoch 4 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: last_changed 2024-05-15T15:55:54.230321+0000 2024-05-15T15:55:55.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: created 2024-05-15T15:24:25.819270+0000 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: min_mon_release 19 (squid) 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: election_strategy: 1 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: 0: [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] mon.a 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: 1: [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] mon.c 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: 2: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.b 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: fsmap 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: osdmap e107: 8 total, 8 up, 8 in 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: mgrmap e49: y(active, since 19s), standbys: x 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='' 2024-05-15T15:55:55.342 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:55.343 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:55 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:55.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: from='client.44136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T15:55:55.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: from='client.44148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: from='client.44160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:55:55.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-05-15T15:55:55.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: 2: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.b 2024-05-15T15:55:55.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: fsmap 2024-05-15T15:55:55.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: osdmap e107: 8 total, 8 up, 8 in 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: mgrmap e49: y(active, since 19s), standbys: x 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: mon.b calling monitor election 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: mon.c calling monitor election 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: mon.a calling monitor election 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: monmap epoch 4 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: fsid 22d82864-12cf-11ef-bc98-c7b262605968 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: last_changed 2024-05-15T15:55:54.230321+0000 2024-05-15T15:55:55.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: created 2024-05-15T15:24:25.819270+0000 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: min_mon_release 19 (squid) 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: election_strategy: 1 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: 0: [v2:172.21.15.43:3300/0,v1:172.21.15.43:6789/0] mon.a 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: 1: [v2:172.21.15.43:3301/0,v1:172.21.15.43:6790/0] mon.c 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: 2: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.b 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: fsmap 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: osdmap e107: 8 total, 8 up, 8 in 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: mgrmap e49: y(active, since 19s), standbys: x 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='' 2024-05-15T15:55:55.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:55.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:55 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:55:56.775 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:55:56] "GET /metrics HTTP/1.1" 200 38001 "" "Prometheus/2.43.0" 2024-05-15T15:55:57.559 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:57 smithi043 ceph-mon[82531]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-05-15T15:55:57.560 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:57 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:57.560 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:57 smithi043 ceph-mon[88099]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-05-15T15:55:57.560 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:57 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:57 smithi053 ceph-mon[67220]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-05-15T15:55:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:57 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:55:57.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:55:57 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:55:57.680+0000 7fbf8c6f9700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-05-15T15:55:59.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:59 smithi043 ceph-mon[88099]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:59.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:59 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:59.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:55:59 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:59 smithi043 ceph-mon[82531]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:59 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:59.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:55:59 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:59.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:59 smithi053 ceph-mon[67220]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:55:59.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:59 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:55:59.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:55:59 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:01.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:00 smithi043 ceph-mon[82531]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:01.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:00 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:01.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:00 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:01.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:00 smithi043 ceph-mon[88099]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:01.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:00 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:01.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:00 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:00 smithi053 ceph-mon[67220]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:00 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:00 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:03.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:02 smithi043 ceph-mon[82531]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:03.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:02 smithi043 ceph-mon[88099]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:03.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:02 smithi053 ceph-mon[67220]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:04.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[82531]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[82531]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:04.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:04.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:04.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:56:04.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[88099]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-05-15T15:56:04.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[88099]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:04.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:04.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:56:04.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:56:04.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:04 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:04 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:56:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:04 smithi053 ceph-mon[67220]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 868 B/s rd, 0 op/s 2024-05-15T15:56:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:04 smithi053 ceph-mon[67220]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:04 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:56:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:56:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:05.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[82531]: Reconfiguring mon.a (monmap changed)... 2024-05-15T15:56:05.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[82531]: Reconfiguring daemon mon.a on smithi043 2024-05-15T15:56:05.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:05.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[88099]: Reconfiguring mon.a (monmap changed)... 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[88099]: Reconfiguring daemon mon.a on smithi043 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:05.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:05.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:05.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:56:05.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:05 smithi053 ceph-mon[67220]: Reconfiguring mon.a (monmap changed)... 2024-05-15T15:56:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:05 smithi053 ceph-mon[67220]: Reconfiguring daemon mon.a on smithi043 2024-05-15T15:56:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:05.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:05.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:56:05.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:06.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:06 smithi043 ceph-mon[82531]: Reconfiguring mgr.y (monmap changed)... 2024-05-15T15:56:06.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:06 smithi043 ceph-mon[82531]: Reconfiguring daemon mgr.y on smithi043 2024-05-15T15:56:06.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:06 smithi043 ceph-mon[82531]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:06.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:56:06] "GET /metrics HTTP/1.1" 200 38007 "" "Prometheus/2.43.0" 2024-05-15T15:56:06.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:06 smithi043 ceph-mon[88099]: Reconfiguring mgr.y (monmap changed)... 2024-05-15T15:56:06.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:06 smithi043 ceph-mon[88099]: Reconfiguring daemon mgr.y on smithi043 2024-05-15T15:56:06.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:06 smithi043 ceph-mon[88099]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:06 smithi053 ceph-mon[67220]: Reconfiguring mgr.y (monmap changed)... 2024-05-15T15:56:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:06 smithi053 ceph-mon[67220]: Reconfiguring daemon mgr.y on smithi043 2024-05-15T15:56:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:06 smithi053 ceph-mon[67220]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:07.493 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:07.493 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:07.494 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:07.494 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:56:07.494 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:56:07.494 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:07.494 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:07.494 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:07.494 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:07.494 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:56:07.494 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:56:07.494 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:07 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:56:07.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:56:07.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:08.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:08 smithi043 ceph-mon[82531]: Reconfiguring mon.c (monmap changed)... 2024-05-15T15:56:08.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:08 smithi043 ceph-mon[82531]: Reconfiguring daemon mon.c on smithi043 2024-05-15T15:56:08.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:08 smithi043 ceph-mon[82531]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:08.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:08 smithi043 ceph-mon[88099]: Reconfiguring mon.c (monmap changed)... 2024-05-15T15:56:08.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:08 smithi043 ceph-mon[88099]: Reconfiguring daemon mon.c on smithi043 2024-05-15T15:56:08.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:08 smithi043 ceph-mon[88099]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:08.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:08 smithi053 ceph-mon[67220]: Reconfiguring mon.c (monmap changed)... 2024-05-15T15:56:08.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:08 smithi053 ceph-mon[67220]: Reconfiguring daemon mon.c on smithi043 2024-05-15T15:56:08.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:08 smithi053 ceph-mon[67220]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:09.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:09.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[82531]: Reconfiguring osd.0 (monmap changed)... 2024-05-15T15:56:09.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-15T15:56:09.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:09.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[82531]: Reconfiguring daemon osd.0 on smithi043 2024-05-15T15:56:09.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[82531]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:09.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:09.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:09.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[88099]: Reconfiguring osd.0 (monmap changed)... 2024-05-15T15:56:09.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-15T15:56:09.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:09.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[88099]: Reconfiguring daemon osd.0 on smithi043 2024-05-15T15:56:09.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:09 smithi043 ceph-mon[88099]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:09 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:09 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:09 smithi053 ceph-mon[67220]: Reconfiguring osd.0 (monmap changed)... 2024-05-15T15:56:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:09 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-15T15:56:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:09 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:09 smithi053 ceph-mon[67220]: Reconfiguring daemon osd.0 on smithi043 2024-05-15T15:56:09.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:09 smithi053 ceph-mon[67220]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:11.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:11.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[82531]: Reconfiguring osd.1 (monmap changed)... 2024-05-15T15:56:11.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-15T15:56:11.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:11.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[82531]: Reconfiguring daemon osd.1 on smithi043 2024-05-15T15:56:11.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:11.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:11.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[88099]: Reconfiguring osd.1 (monmap changed)... 2024-05-15T15:56:11.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-15T15:56:11.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:11.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:11 smithi043 ceph-mon[88099]: Reconfiguring daemon osd.1 on smithi043 2024-05-15T15:56:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:11 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:11 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:11 smithi053 ceph-mon[67220]: Reconfiguring osd.1 (monmap changed)... 2024-05-15T15:56:11.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:11 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-15T15:56:11.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:11 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:11.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:11 smithi053 ceph-mon[67220]: Reconfiguring daemon osd.1 on smithi043 2024-05-15T15:56:12.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:12 smithi043 ceph-mon[82531]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:12.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:12.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:12.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-15T15:56:12.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:12.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:12 smithi043 ceph-mon[88099]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:12.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:12.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:12.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-15T15:56:12.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:12 smithi053 ceph-mon[67220]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-15T15:56:12.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:13.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:13 smithi043 ceph-mon[82531]: Reconfiguring osd.2 (monmap changed)... 2024-05-15T15:56:13.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:13 smithi043 ceph-mon[82531]: Reconfiguring daemon osd.2 on smithi043 2024-05-15T15:56:13.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:13 smithi043 ceph-mon[88099]: Reconfiguring osd.2 (monmap changed)... 2024-05-15T15:56:13.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:13 smithi043 ceph-mon[88099]: Reconfiguring daemon osd.2 on smithi043 2024-05-15T15:56:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:13 smithi053 ceph-mon[67220]: Reconfiguring osd.2 (monmap changed)... 2024-05-15T15:56:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:13 smithi053 ceph-mon[67220]: Reconfiguring daemon osd.2 on smithi043 2024-05-15T15:56:14.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:14 smithi043 ceph-mon[82531]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:56:14.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:14 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:14.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:14 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:14.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:14 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-15T15:56:14.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:14 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:14.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:14 smithi043 ceph-mon[88099]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:56:14.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:14 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:14.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:14 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:14.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:14 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-15T15:56:14.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:14 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:14.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:14 smithi053 ceph-mon[67220]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:56:14.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:14 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:14.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:14 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:14.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:14 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-15T15:56:14.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:14 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:15 smithi043 ceph-mon[82531]: Reconfiguring osd.3 (monmap changed)... 2024-05-15T15:56:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:15 smithi043 ceph-mon[82531]: Reconfiguring daemon osd.3 on smithi043 2024-05-15T15:56:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:15 smithi043 ceph-mon[88099]: Reconfiguring osd.3 (monmap changed)... 2024-05-15T15:56:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:15 smithi043 ceph-mon[88099]: Reconfiguring daemon osd.3 on smithi043 2024-05-15T15:56:15.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:15 smithi053 ceph-mon[67220]: Reconfiguring osd.3 (monmap changed)... 2024-05-15T15:56:15.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:15 smithi053 ceph-mon[67220]: Reconfiguring daemon osd.3 on smithi043 2024-05-15T15:56:16.531 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:16 smithi053 ceph-mon[67220]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:16.531 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:16 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:16.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:16 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:16.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:16 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:16.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:16 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:16.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:16 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:16.575 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[82531]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:16.576 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:16.576 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:16.576 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:16.576 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:16.576 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:16.576 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[88099]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:16.576 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:16.576 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:16.576 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:16.577 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:16.577 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:16 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:16.577 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:56:16] "GET /metrics HTTP/1.1" 200 38007 "" "Prometheus/2.43.0" 2024-05-15T15:56:17.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:17 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:17.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:17 smithi043 ceph-mon[82531]: Reconfiguring rgw.foo.smithi043.fiueec (monmap changed)... 2024-05-15T15:56:17.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:17 smithi043 ceph-mon[82531]: Reconfiguring daemon rgw.foo.smithi043.fiueec on smithi043 2024-05-15T15:56:17.318 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:17 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:17.318 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:17 smithi043 ceph-mon[88099]: Reconfiguring rgw.foo.smithi043.fiueec (monmap changed)... 2024-05-15T15:56:17.318 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:17 smithi043 ceph-mon[88099]: Reconfiguring daemon rgw.foo.smithi043.fiueec on smithi043 2024-05-15T15:56:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:17 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:17 smithi053 ceph-mon[67220]: Reconfiguring rgw.foo.smithi043.fiueec (monmap changed)... 2024-05-15T15:56:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:17 smithi053 ceph-mon[67220]: Reconfiguring daemon rgw.foo.smithi043.fiueec on smithi043 2024-05-15T15:56:18.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[82531]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:18.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[88099]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:18 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:18.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:18 smithi053 ceph-mon[67220]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:18.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:18 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:18.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:18 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:18.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:18 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:18.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:18 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:18.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:18 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:19.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:19 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:56:19.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:19 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:56:19.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:19 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:56:19.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:19 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:56:19.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:56:19.293+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:19.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:56:19.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:19.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:19.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:19.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:19.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:19.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:56:19.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:19.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:56:19.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:19.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:56:19.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:19.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:19.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:19.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:19.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:19.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:19.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:19.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:19.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:19.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:56:19.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:19.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:19.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:19.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:19.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:19.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:19.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:19.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:56:19.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:19.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:19.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:19.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:19.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:19.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:19.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:19.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:19.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:19 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:19 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:56:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:19 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:56:20.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:20 smithi043 ceph-mon[88099]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:20.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:56:20.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:56:20.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:20.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:20.525 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:20 smithi043 ceph-mon[82531]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:20.526 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:56:20.526 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:56:20.526 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:20.526 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:20.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:20 smithi053 ceph-mon[67220]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:20.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-15T15:56:20.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-15T15:56:20.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:20.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:21.227 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:21 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:21.227 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:21.227 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:21.227 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:21.227 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:21.227 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.227 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.227 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.228 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:56:21.229 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:21.230 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.231 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:21 smithi053 ceph-mon[67220]: Reconfiguring mon.b (monmap changed)... 2024-05-15T15:56:21.232 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:21 smithi053 ceph-mon[67220]: Reconfiguring daemon mon.b on smithi053 2024-05-15T15:56:21.232 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:21 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:21.232 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:21 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:21.232 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:21 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:21.232 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:21 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:21.232 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:21 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:56:21.232 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:21 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:21.492 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:21.492 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:21.492 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:21.493 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:21.494 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:21.495 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:21.496 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:21.497 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:21.497 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:21.497 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:21.497 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:21.497 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:21.497 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.497 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.497 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:21.497 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:21.497 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:21.498 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:21.499 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:21.500 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:21.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:21.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:21.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:21.501 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:21.501 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:21.501 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:21.501 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:21.501 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.501 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[82531]: Reconfiguring mon.b (monmap changed)... 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[82531]: Reconfiguring daemon mon.b on smithi053 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:21.502 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:21.503 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:21.503 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:21.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[88099]: Reconfiguring mon.b (monmap changed)... 2024-05-15T15:56:21.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[88099]: Reconfiguring daemon mon.b on smithi053 2024-05-15T15:56:21.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:21.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:21.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:21.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-15T15:56:21.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-15T15:56:21.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:21 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:22.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:22 smithi043 ceph-mon[82531]: Reconfiguring mgr.x (monmap changed)... 2024-05-15T15:56:22.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:22 smithi043 ceph-mon[82531]: Reconfiguring daemon mgr.x on smithi053 2024-05-15T15:56:22.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:22 smithi043 ceph-mon[82531]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:22.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:22 smithi043 ceph-mon[88099]: Reconfiguring mgr.x (monmap changed)... 2024-05-15T15:56:22.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:22 smithi043 ceph-mon[88099]: Reconfiguring daemon mgr.x on smithi053 2024-05-15T15:56:22.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:22 smithi043 ceph-mon[88099]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:22.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:22 smithi053 ceph-mon[67220]: Reconfiguring mgr.x (monmap changed)... 2024-05-15T15:56:22.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:22 smithi053 ceph-mon[67220]: Reconfiguring daemon mgr.x on smithi053 2024-05-15T15:56:22.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:22 smithi053 ceph-mon[67220]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:22.682 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (3m) 24s ago 25m 19.1M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (2m) 43s ago 25m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 24s ago 24m 64.8M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (4m) 43s ago 30m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (14m) 24s ago 31m 497M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (58s) 24s ago 31m 39.4M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (107s) 43s ago 30m 38.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (31s) 24s ago 30m 20.4M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (4m) 24s ago 26m 8040k - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (4m) 43s ago 26m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:56:23.379 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (29m) 24s ago 29m 54.1M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:56:23.380 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (29m) 24s ago 29m 58.2M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:56:23.380 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (28m) 24s ago 28m 50.5M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:56:23.380 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (28m) 24s ago 28m 53.6M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:56:23.380 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (28m) 43s ago 28m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:56:23.380 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (27m) 43s ago 27m 51.9M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:56:23.380 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (27m) 43s ago 27m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:56:23.380 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (27m) 43s ago 27m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:56:23.380 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (3m) 43s ago 26m 68.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:56:23.380 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (24m) 24s ago 24m 77.6M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:56:23.380 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (24m) 43s ago 24m 76.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:56:23.798 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:56:23.798 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:56:23.798 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T15:56:23.798 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:56:23.798 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:56:23.798 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:56:23.798 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 5 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:56:23.799 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:56:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[88099]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[88099]: Reconfiguring osd.4 (monmap changed)... 2024-05-15T15:56:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-15T15:56:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[88099]: Reconfiguring daemon osd.4 on smithi053 2024-05-15T15:56:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[88099]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:24.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[88099]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:24.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[82531]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:24.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:24.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:24.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[82531]: Reconfiguring osd.4 (monmap changed)... 2024-05-15T15:56:24.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-15T15:56:24.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:24.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[82531]: Reconfiguring daemon osd.4 on smithi053 2024-05-15T15:56:24.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[82531]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:24.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:23 smithi043 ceph-mon[82531]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:24.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:23 smithi053 ceph-mon[67220]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:24.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:24.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:24.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:23 smithi053 ceph-mon[67220]: Reconfiguring osd.4 (monmap changed)... 2024-05-15T15:56:24.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-15T15:56:24.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:24.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:23 smithi053 ceph-mon[67220]: Reconfiguring daemon osd.4 on smithi053 2024-05-15T15:56:24.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:23 smithi053 ceph-mon[67220]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:24.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:23 smithi053 ceph-mon[67220]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:24.147 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:56:24.147 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T15:56:24.147 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:56:24.147 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi043", 2024-05-15T15:56:24.147 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [ 2024-05-15T15:56:24.147 INFO:teuthology.orchestra.run.smithi043.stdout: "mon" 2024-05-15T15:56:24.147 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T15:56:24.148 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "2/2 daemons upgraded", 2024-05-15T15:56:24.148 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading mon daemons", 2024-05-15T15:56:24.148 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:56:24.148 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:56:25.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[82531]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:25.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/755771156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:25.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[82531]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:25.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:25.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:25.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-15T15:56:25.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:25.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[88099]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:25.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/755771156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:25.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[88099]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:25.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:25.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:25.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-15T15:56:25.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:25.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:24 smithi053 ceph-mon[67220]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:25.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:24 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/755771156' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:25.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:24 smithi053 ceph-mon[67220]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:25.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:25.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:25.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-15T15:56:25.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:25 smithi053 ceph-mon[67220]: Reconfiguring osd.5 (monmap changed)... 2024-05-15T15:56:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:25 smithi053 ceph-mon[67220]: Reconfiguring daemon osd.5 on smithi053 2024-05-15T15:56:25.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:25 smithi053 ceph-mon[67220]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:26.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:25 smithi043 ceph-mon[82531]: Reconfiguring osd.5 (monmap changed)... 2024-05-15T15:56:26.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:25 smithi043 ceph-mon[82531]: Reconfiguring daemon osd.5 on smithi053 2024-05-15T15:56:26.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:25 smithi043 ceph-mon[82531]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:26.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:25 smithi043 ceph-mon[88099]: Reconfiguring osd.5 (monmap changed)... 2024-05-15T15:56:26.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:25 smithi043 ceph-mon[88099]: Reconfiguring daemon osd.5 on smithi053 2024-05-15T15:56:26.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:25 smithi043 ceph-mon[88099]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:26.768 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:56:26] "GET /metrics HTTP/1.1" 200 38008 "" "Prometheus/2.43.0" 2024-05-15T15:56:27.026 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:26 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:27.026 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:26 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:27.026 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:26 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:27.026 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:26 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-15T15:56:27.026 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:26 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:27.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:26 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:27.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:26 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:27.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:26 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:27.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:26 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-15T15:56:27.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:26 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:27.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:26 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:27.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:26 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:27.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:26 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:27.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:26 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-15T15:56:27.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:26 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:28.037 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:27 smithi053 ceph-mon[67220]: Reconfiguring osd.6 (monmap changed)... 2024-05-15T15:56:28.038 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:27 smithi053 ceph-mon[67220]: Reconfiguring daemon osd.6 on smithi053 2024-05-15T15:56:28.038 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:27 smithi053 ceph-mon[67220]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:27 smithi043 ceph-mon[82531]: Reconfiguring osd.6 (monmap changed)... 2024-05-15T15:56:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:27 smithi043 ceph-mon[82531]: Reconfiguring daemon osd.6 on smithi053 2024-05-15T15:56:28.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:27 smithi043 ceph-mon[82531]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:28.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:27 smithi043 ceph-mon[88099]: Reconfiguring osd.6 (monmap changed)... 2024-05-15T15:56:28.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:27 smithi043 ceph-mon[88099]: Reconfiguring daemon osd.6 on smithi053 2024-05-15T15:56:28.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:27 smithi043 ceph-mon[88099]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:29 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:29.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:29 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:29.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:29 smithi053 ceph-mon[67220]: Reconfiguring osd.7 (monmap changed)... 2024-05-15T15:56:29.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:29 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-15T15:56:29.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:29 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:29.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:29 smithi053 ceph-mon[67220]: Reconfiguring daemon osd.7 on smithi053 2024-05-15T15:56:29.389 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:29.389 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:29.389 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[88099]: Reconfiguring osd.7 (monmap changed)... 2024-05-15T15:56:29.389 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-15T15:56:29.389 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:29.389 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[88099]: Reconfiguring daemon osd.7 on smithi053 2024-05-15T15:56:29.390 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:29.390 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:29.390 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[82531]: Reconfiguring osd.7 (monmap changed)... 2024-05-15T15:56:29.390 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-15T15:56:29.390 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:29.390 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:29 smithi043 ceph-mon[82531]: Reconfiguring daemon osd.7 on smithi053 2024-05-15T15:56:30.419 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:30 smithi053 ceph-mon[67220]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:30.420 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:30 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:30.420 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:30 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:30.420 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:30 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:30.420 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:30 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:30.420 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:30 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:30.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[82531]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:30.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:30.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:30.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:30.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:30.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:30.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[88099]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:30.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:30.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:30.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:30.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T15:56:30.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:30 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:31.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:31 smithi043 ceph-mon[82531]: Reconfiguring rgw.foo.smithi053.vjmwpu (monmap changed)... 2024-05-15T15:56:31.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:31 smithi043 ceph-mon[82531]: Reconfiguring daemon rgw.foo.smithi053.vjmwpu on smithi053 2024-05-15T15:56:31.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:31 smithi043 ceph-mon[88099]: Reconfiguring rgw.foo.smithi053.vjmwpu (monmap changed)... 2024-05-15T15:56:31.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:31 smithi043 ceph-mon[88099]: Reconfiguring daemon rgw.foo.smithi053.vjmwpu on smithi053 2024-05-15T15:56:31.395 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:31 smithi053 ceph-mon[67220]: Reconfiguring rgw.foo.smithi053.vjmwpu (monmap changed)... 2024-05-15T15:56:31.395 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:31 smithi053 ceph-mon[67220]: Reconfiguring daemon rgw.foo.smithi053.vjmwpu on smithi053 2024-05-15T15:56:32.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:32.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:56:32.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.589 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:32.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.592 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.593 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.594 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.594 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.594 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.594 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.594 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.594 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:56:32.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:56:32.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.597 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.598 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.598 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.598 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.598 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.598 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.598 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.598 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.598 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.599 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.600 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.601 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.601 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:56:32.601 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:56:32.601 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:56:32.601 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.602 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:56:32.603 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:56:32.604 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.605 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.606 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.606 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.606 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.606 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.606 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:56:32.606 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:56:32.606 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:56:32.606 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:32 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:56:32.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:32.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:56:32.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:56:32.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:56:32.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:56:32.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:56:32.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:56:32.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:56:32.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T15:56:32.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:56:32.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:56:32.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:56:32.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:56:32.636 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:56:32.637 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:56:32.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:56:32.639 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:56:32.639 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:56:32.639 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:32 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mon 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all crash 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mds 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all nfs 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all node-exporter 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all prometheus 2024-05-15T15:56:33.320 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all alertmanager 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all grafana 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all loki 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all promtail 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Finalizing container_image settings 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: Upgrade: Complete! 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 930 B/s rd, 0 op/s 2024-05-15T15:56:33.321 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mon 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all crash 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mds 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T15:56:33.322 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all nfs 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all node-exporter 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all prometheus 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all alertmanager 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all grafana 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all loki 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all promtail 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Finalizing container_image settings 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: Upgrade: Complete! 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:33.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:56:33.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 930 B/s rd, 0 op/s 2024-05-15T15:56:33.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:56:33.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:33.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:33.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:33 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:33.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mon 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all crash 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mds 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all nfs 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all node-exporter 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all prometheus 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all alertmanager 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all grafana 2024-05-15T15:56:33.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all loki 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all promtail 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Finalizing container_image settings 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: Upgrade: Complete! 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 930 B/s rd, 0 op/s 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:33.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:33.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:33.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:33 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:34.237 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:56:34.090+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:56:34.238 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:34.239 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:34.239 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:34.239 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:34.239 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:34.239 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:34.239 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:34.239 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:34.240 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:34.240 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:56:34.240 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:34.240 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:34.240 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:34.240 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:34.240 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:34.240 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:34.240 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:34.240 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:56:34.241 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:34.241 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:56:34.241 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:34.241 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:56:34.241 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:34.241 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:56:34.241 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:34.241 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:34.241 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:34.242 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:34.243 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:34.243 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:34.243 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:34.243 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:34.243 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:34.243 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:34.243 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:34.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:56:34.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:56:34.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:56:34.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:34.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:56:34.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:56:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:56:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:56:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:56:34.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:34 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:34.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:34 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:56:34.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:34 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:56:34.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:34 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:56:34.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:34 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:34.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:34 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:56:34.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:34 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:35.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:35.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:56:35.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:35.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:35.591 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:35.591 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:35.591 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:35.591 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:35.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:35.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:35.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:35.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:35.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:56:35.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:35.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:35.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[88099]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[88099]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:35.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[82531]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[82531]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:56:35.597 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:35.598 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:35.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:35 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:35.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:35.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:35.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:35.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:35.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:35.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:35 smithi053 ceph-mon[67220]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:56:35.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:35 smithi053 ceph-mon[67220]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-05-15T15:56:35.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:35 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:56:35.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:35.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:56:35.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:56:35.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:35 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:56:35.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:35.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:56:36.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:56:35.958+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:36.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:56:36.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:36.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:36.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:36.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:36.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:36.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:56:36.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:36.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:56:36.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:36.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:56:36.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:36.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:36.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:36.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:36.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:36.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:36.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:36.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:56:36.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:36.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:35 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:36.801 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:56:36] "GET /metrics HTTP/1.1" 200 38010 "" "Prometheus/2.43.0" 2024-05-15T15:56:37.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:36 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:36 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:37.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:37.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:37.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:37.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:37.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:37.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:37.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:37.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:37.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:37.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:36 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:37.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:36 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:37.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:37.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:37.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:37.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:37.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:36 smithi043 ceph-mon[88099]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-15T15:56:37.096 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:37.096 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:37.096 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:37.096 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.096 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:36 smithi043 ceph-mon[82531]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-15T15:56:37.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:36 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:37.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:36 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:37.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:37.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:37.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:37.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:37.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:37.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:56:37.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:56:37.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:56:37.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:56:37.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:56:37.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:56:37.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:56:37.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:56:37.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:56:37.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:56:37.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:56:37.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:56:37.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:56:37.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:36 smithi053 ceph-mon[67220]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-15T15:56:39.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:39 smithi043 ceph-mon[82531]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-05-15T15:56:39.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:39 smithi043 ceph-mon[88099]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-05-15T15:56:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:39 smithi053 ceph-mon[67220]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-05-15T15:56:41.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:41 smithi043 ceph-mon[82531]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-05-15T15:56:41.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:41 smithi043 ceph-mon[88099]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-05-15T15:56:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:41 smithi053 ceph-mon[67220]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 650 B/s rd, 0 op/s 2024-05-15T15:56:43.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:43 smithi043 ceph-mon[82531]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:43.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:43 smithi043 ceph-mon[88099]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:43 smithi053 ceph-mon[67220]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:56:45.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:45 smithi043 ceph-mon[82531]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:56:45.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:45 smithi043 ceph-mon[88099]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:56:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:45 smithi053 ceph-mon[67220]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:56:46.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:56:46] "GET /metrics HTTP/1.1" 200 38010 "" "Prometheus/2.43.0" 2024-05-15T15:56:47.432 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:47 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:47.432 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:47 smithi053 ceph-mon[67220]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:47.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:47 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:47.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:47 smithi043 ceph-mon[82531]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:47 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:47 smithi043 ceph-mon[88099]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:49 smithi043 ceph-mon[82531]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:49.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:49 smithi043 ceph-mon[88099]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:49 smithi053 ceph-mon[67220]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:50.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:50 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:50.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:50 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:50 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:56:51.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:51 smithi043 ceph-mon[88099]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:51.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:51 smithi043 ceph-mon[82531]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:51 smithi053 ceph-mon[67220]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:53.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:53 smithi043 ceph-mon[88099]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:53.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:53 smithi043 ceph-mon[82531]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:53.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:53 smithi053 ceph-mon[67220]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:55.209 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T15:56:55.535 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:55 smithi043 ceph-mon[88099]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:55.535 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:55 smithi043 ceph-mon[82531]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:55.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:55 smithi053 ceph-mon[67220]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:56.525 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:56 smithi043 ceph-mon[82531]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:56.525 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:56:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:56:56] "GET /metrics HTTP/1.1" 200 38005 "" "Prometheus/2.43.0" 2024-05-15T15:56:56.525 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:56 smithi043 ceph-mon[88099]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:56.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:56 smithi053 ceph-mon[67220]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:57.233 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:56:57.233 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (3m) 58s ago 26m 19.1M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:56:57.233 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (3m) 77s ago 25m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:56:57.233 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 58s ago 25m 64.8M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:56:57.233 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (5m) 77s ago 30m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:56:57.233 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (15m) 58s ago 32m 497M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:56:57.233 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (91s) 58s ago 32m 39.4M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T15:56:57.233 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (2m) 77s ago 30m 38.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:56:57.233 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (64s) 58s ago 31m 20.4M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T15:56:57.233 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (4m) 58s ago 26m 8040k - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (4m) 77s ago 26m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (30m) 58s ago 30m 54.1M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (29m) 58s ago 29m 58.2M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (29m) 58s ago 29m 50.5M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (29m) 58s ago 29m 53.6M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (28m) 77s ago 28m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (28m) 77s ago 28m 51.9M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (27m) 77s ago 27m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (27m) 77s ago 27m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (4m) 77s ago 26m 68.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:56:57.234 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (25m) 58s ago 25m 77.6M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:56:57.235 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (25m) 77s ago 25m 76.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:56:57.532 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:57 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:57.533 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:57 smithi043 ceph-mon[82531]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:57.533 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:57 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:57.533 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:57 smithi043 ceph-mon[88099]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:57 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:56:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:57 smithi053 ceph-mon[67220]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:56:57.834 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-05-15T15:56:58.486 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:58 smithi043 ceph-mon[82531]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:58.486 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:58 smithi043 ceph-mon[88099]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:58 smithi053 ceph-mon[67220]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:56:59.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:56:59 smithi043 ceph-mon[82531]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:59.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:56:59 smithi043 ceph-mon[88099]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:59.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:56:59 smithi053 ceph-mon[67220]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:56:59.897 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:57:00.456 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:00 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/766465165' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:00.456 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:00 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/766465165' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:00.483 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-05-15T15:57:00.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:00 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/766465165' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:01.472 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:01 smithi043 ceph-mon[82531]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:01.472 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:01 smithi043 ceph-mon[88099]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:01 smithi053 ceph-mon[67220]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:02.545 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)" 2024-05-15T15:57:03.072 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-05-15T15:57:03.399 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:03 smithi043 ceph-mon[82531]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:03.399 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:03 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3663345191' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:03.399 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:03 smithi043 ceph-mon[88099]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:03.399 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:03 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3663345191' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:03 smithi053 ceph-mon[67220]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:03 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3663345191' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:05.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:05 smithi043 ceph-mon[88099]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:05.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:05.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:05 smithi043 ceph-mon[82531]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:05.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:05 smithi053 ceph-mon[67220]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:06.322 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:06 smithi043 ceph-mon[82531]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:06.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:06 smithi043 ceph-mon[88099]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:06.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:57:06] "GET /metrics HTTP/1.1" 200 38007 "" "Prometheus/2.43.0" 2024-05-15T15:57:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:06 smithi053 ceph-mon[67220]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:07.116 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:57:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:07 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:07 smithi043 ceph-mon[82531]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:07 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:07 smithi043 ceph-mon[88099]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:07 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:07 smithi053 ceph-mon[67220]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:07.705 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade status' 2024-05-15T15:57:09.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:09 smithi043 ceph-mon[82531]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:09.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:09 smithi043 ceph-mon[88099]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:09.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:09 smithi053 ceph-mon[67220]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:09.685 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:57:09.685 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": null, 2024-05-15T15:57:09.685 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": false, 2024-05-15T15:57:09.685 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "", 2024-05-15T15:57:09.685 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:57:09.685 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": null, 2024-05-15T15:57:09.685 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "", 2024-05-15T15:57:09.686 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:57:09.686 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:57:10.311 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T15:57:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:11 smithi043 ceph-mon[88099]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:11.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:11 smithi043 ceph-mon[88099]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:11.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:11 smithi043 ceph-mon[82531]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:11.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:11 smithi043 ceph-mon[82531]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:11 smithi053 ceph-mon[67220]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:11.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:11 smithi053 ceph-mon[67220]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:12.365 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:57:12.365 INFO:teuthology.orchestra.run.smithi043.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:57:12.365 INFO:teuthology.orchestra.run.smithi043.stdout: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:57:13.030 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-05-15T15:57:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:13 smithi043 ceph-mon[82531]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:13 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3056161856' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:57:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:13 smithi043 ceph-mon[88099]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:13 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3056161856' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:57:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:13 smithi053 ceph-mon[67220]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:13.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:13 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3056161856' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:57:15.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:15 smithi043 ceph-mon[82531]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:15.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:15 smithi043 ceph-mon[88099]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:15.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:15 smithi053 ceph-mon[67220]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:57:16.286 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:16 smithi043 ceph-mon[82531]: from='client.44238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:16.287 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:16 smithi043 ceph-mon[88099]: from='client.44238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:16.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:57:16] "GET /metrics HTTP/1.1" 200 38007 "" "Prometheus/2.43.0" 2024-05-15T15:57:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:16 smithi053 ceph-mon[67220]: from='client.44238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:17.165 INFO:teuthology.orchestra.run.smithi043.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:57:17.490 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:17 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:17.490 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:17 smithi043 ceph-mon[82531]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:17.490 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:17 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:17.491 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:17 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:17.491 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:17 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:17.491 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:17 smithi043 ceph-mon[88099]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:17.491 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:17 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:17.491 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:17 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:17.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:17 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:17.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:17 smithi053 ceph-mon[67220]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:17.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:17 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:17.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:17 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:17.766 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-15T15:57:18.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[88099]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:57:18.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:57:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:18.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[82531]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:57:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:57:18.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:18.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:18.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:18.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:18 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:18.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:18 smithi053 ceph-mon[67220]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:57:18.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:18 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:18.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:18 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:57:18.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:18 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:18.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:18 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:18.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:18 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:18.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:18 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:19.300 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:19 smithi043 ceph-mon[82531]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-05-15T15:57:19.301 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:19 smithi043 ceph-mon[82531]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:19.301 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:19 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:57:19.301 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:19 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:57:19.301 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:19 smithi043 ceph-mon[88099]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-05-15T15:57:19.301 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:19 smithi043 ceph-mon[88099]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:19.301 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:19 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:57:19.301 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:19 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:57:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:19 smithi053 ceph-mon[67220]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-05-15T15:57:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:19 smithi053 ceph-mon[67220]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:19.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:19 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:57:19.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:19 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:57:19.803 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:57:20.503 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:20.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:20.503 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (4m) 81s ago 26m 19.1M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (3m) 101s ago 26m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 81s ago 25m 64.8M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (5m) 101s ago 31m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (15m) 81s ago 32m 497M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (115s) 81s ago 32m 39.4M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (2m) 101s ago 31m 38.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (88s) 81s ago 31m 20.4M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (5m) 81s ago 27m 8040k - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (5m) 101s ago 27m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (30m) 81s ago 30m 54.1M 2993M 17.2.0 e1d6a67b021e c0afad6ff420 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (30m) 81s ago 30m 58.2M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (29m) 81s ago 29m 50.5M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:57:20.504 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (29m) 81s ago 29m 53.6M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:57:20.505 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (29m) 101s ago 29m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:57:20.505 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (28m) 101s ago 28m 51.9M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:57:20.505 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (28m) 101s ago 28m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:57:20.505 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (27m) 101s ago 27m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:57:20.505 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (4m) 101s ago 27m 68.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:57:20.505 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (25m) 81s ago 25m 77.6M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:57:20.505 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (25m) 101s ago 25m 76.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:57:20.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:20.949 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:57:20.950 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:57:20.951 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:57:20.951 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-15T15:57:20.951 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 5 2024-05-15T15:57:20.951 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:57:20.951 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:57:21.245 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:57:21.006+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:21.245 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:57:21.246 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:21.247 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:21.247 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:21.247 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:21.247 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:21.247 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:21.247 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:21.247 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:57:21.248 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:21.249 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:21.250 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:21 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:21.316 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:57:21.316 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", 2024-05-15T15:57:21.316 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:57:21.316 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-05-15T15:57:21.316 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:57:21.317 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "", 2024-05-15T15:57:21.317 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 image", 2024-05-15T15:57:21.317 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:57:21.317 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:57:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:21 smithi043 ceph-mon[88099]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:21 smithi043 ceph-mon[88099]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:21 smithi043 ceph-mon[88099]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:21.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:21 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3594180528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:21.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:21 smithi043 ceph-mon[82531]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:21.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:21 smithi043 ceph-mon[82531]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:21.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:21 smithi043 ceph-mon[82531]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:21.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:21 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3594180528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:21 smithi053 ceph-mon[67220]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:21 smithi053 ceph-mon[67220]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:21 smithi053 ceph-mon[67220]: from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:21.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:21 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3594180528' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:22.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:22 smithi043 ceph-mon[82531]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:22.502 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:22 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:22.502 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:22.502 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:22.502 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:22.502 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:22.502 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.502 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.502 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:22.502 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:57:22.503 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:57:22.504 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:57:22.505 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:22.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:22.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:22.506 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:22.506 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:22 smithi043 ceph-mon[88099]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:22.506 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:22 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:22.506 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:22.506 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:22.507 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:57:22.508 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:22.509 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:22.510 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:22.511 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:22.511 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:22.511 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:22.511 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:22.511 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.511 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.511 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:22 smithi043 ceph-mon[82531]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:57:22.512 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:22.512 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:22.512 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:22.512 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.512 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.512 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:22 smithi043 ceph-mon[88099]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:57:22.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:22 smithi053 ceph-mon[67220]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:22.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:22 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:22.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:22.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:22.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:22.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:22.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:57:22.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:22.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:22.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:22.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:57:22.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:22.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:22.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:22.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:22.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:22.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:22.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:22.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:57:22.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:57:22.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:57:22.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:57:22.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:57:22.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:57:22.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:22.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:22.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:22.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:22.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:22.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:57:22.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:22.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:22.636 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:22.637 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:57:22.637 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:22.637 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:22.637 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:22.637 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:22.637 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:22.638 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:22.638 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:22.638 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:22.638 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:22.638 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:22 smithi053 ceph-mon[67220]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:57:23.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:23 smithi043 ceph-mon[88099]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:23.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:23 smithi043 ceph-mon[88099]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:23.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:23 smithi043 ceph-mon[82531]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:23.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:23 smithi043 ceph-mon[82531]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:23.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:23 smithi053 ceph-mon[67220]: from='client.54205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:23.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:23 smithi053 ceph-mon[67220]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:24.703 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.703 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:57:24.703 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:57:24.703 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:24.703 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:24.703 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mgr 2024-05-15T15:57:24.703 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.703 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:24.703 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mon 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all crash 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: Upgrade: osd.0 is safe to restart 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[82531]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:57:24.704 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mgr 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mon 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all crash 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: Upgrade: osd.0 is safe to restart 2024-05-15T15:57:24.705 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:24 smithi043 ceph-mon[88099]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:24.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mgr 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mon 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all crash 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:24.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-15T15:57:24.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-15T15:57:24.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: Upgrade: osd.0 is safe to restart 2024-05-15T15:57:24.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:24 smithi053 ceph-mon[67220]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:26 smithi043 ceph-mon[82531]: Upgrade: Updating osd.0 2024-05-15T15:57:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:26 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:26 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-15T15:57:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:26 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:26 smithi043 ceph-mon[82531]: Deploying daemon osd.0 on smithi043 2024-05-15T15:57:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:26 smithi043 ceph-mon[88099]: Upgrade: Updating osd.0 2024-05-15T15:57:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:26 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:26 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-15T15:57:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:26 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:26 smithi043 ceph-mon[88099]: Deploying daemon osd.0 on smithi043 2024-05-15T15:57:26.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:26 smithi053 ceph-mon[67220]: Upgrade: Updating osd.0 2024-05-15T15:57:26.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:26 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:26.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:26 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-15T15:57:26.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:26 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:26.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:26 smithi053 ceph-mon[67220]: Deploying daemon osd.0 on smithi043 2024-05-15T15:57:26.610 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:57:26] "GET /metrics HTTP/1.1" 200 38004 "" "Prometheus/2.43.0" 2024-05-15T15:57:27.334 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:27 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:27.334 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:27 smithi043 ceph-mon[82531]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:27.334 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:27 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:27.334 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:27 smithi043 ceph-mon[88099]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:27.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:27 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:27.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:27 smithi053 ceph-mon[67220]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:57:28.586 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:28 smithi043 systemd[1]: Stopping Ceph osd.0 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:57:29.052 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[31968]: 2024-05-15T15:57:28.596+0000 7fae84dbe700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T15:57:29.052 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[31968]: 2024-05-15T15:57:28.596+0000 7fae84dbe700 -1 osd.0 107 *** Got signal Terminated *** 2024-05-15T15:57:29.052 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:28 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[31968]: 2024-05-15T15:57:28.596+0000 7fae84dbe700 -1 osd.0 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T15:57:29.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:29 smithi043 ceph-mon[82531]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:57:29.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:29 smithi043 ceph-mon[82531]: osd.0 marked itself down and dead 2024-05-15T15:57:29.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:29 smithi043 ceph-mon[88099]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:57:29.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:29 smithi043 ceph-mon[88099]: osd.0 marked itself down and dead 2024-05-15T15:57:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:29 smithi053 ceph-mon[67220]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:57:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:29 smithi053 ceph-mon[67220]: osd.0 marked itself down and dead 2024-05-15T15:57:29.789 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:29 smithi043 podman[95501]: 2024-05-15 15:57:29.518445827 +0000 UTC m=+1.036846059 container died c0afad6ff420757e1bc4aa64726458ff387bb41b2f6e2435a9d2f6fb012a0669 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, version=8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, name=centos-stream, architecture=x86_64, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.component=centos-stream-container) 2024-05-15T15:57:30.061 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:29 smithi043 podman[95501]: 2024-05-15 15:57:29.886898185 +0000 UTC m=+1.405298418 container cleanup c0afad6ff420757e1bc4aa64726458ff387bb41b2f6e2435a9d2f6fb012a0669 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, release=754, name=centos-stream, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, distribution-scope=public, version=8, RELEASE=HEAD) 2024-05-15T15:57:30.061 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:29 smithi043 bash[95501]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0 2024-05-15T15:57:30.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:30 smithi043 ceph-mon[88099]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:57:30.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:30 smithi043 ceph-mon[88099]: osdmap e108: 8 total, 7 up, 8 in 2024-05-15T15:57:30.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:30 smithi043 ceph-mon[82531]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:57:30.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:30 smithi043 ceph-mon[82531]: osdmap e108: 8 total, 7 up, 8 in 2024-05-15T15:57:30.337 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:30 smithi043 podman[95517]: 2024-05-15 15:57:30.180192696 +0000 UTC m=+0.658042301 container remove c0afad6ff420757e1bc4aa64726458ff387bb41b2f6e2435a9d2f6fb012a0669 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , name=centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD) 2024-05-15T15:57:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:30 smithi053 ceph-mon[67220]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:57:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:30 smithi053 ceph-mon[67220]: osdmap e108: 8 total, 7 up, 8 in 2024-05-15T15:57:30.777 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:30 smithi043 podman[95604]: 2024-05-15 15:57:30.428258572 +0000 UTC m=+0.018119578 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:57:30.777 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:30 smithi043 podman[95604]: 2024-05-15 15:57:30.556036477 +0000 UTC m=+0.145897461 container create bfb528c26072bc42753ff7e23649bc34332c78ecaf396acb5a80eb459c7b31f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T15:57:30.777 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:30 smithi043 podman[95604]: 2024-05-15 15:57:30.775535714 +0000 UTC m=+0.365396722 container init bfb528c26072bc42753ff7e23649bc34332c78ecaf396acb5a80eb459c7b31f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d) 2024-05-15T15:57:31.062 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:30 smithi043 podman[95604]: 2024-05-15 15:57:30.779549521 +0000 UTC m=+0.369410521 container start bfb528c26072bc42753ff7e23649bc34332c78ecaf396acb5a80eb459c7b31f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T15:57:31.062 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:30 smithi043 podman[95604]: 2024-05-15 15:57:30.90029722 +0000 UTC m=+0.490158224 container attach bfb528c26072bc42753ff7e23649bc34332c78ecaf396acb5a80eb459c7b31f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-05-15T15:57:31.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:31 smithi043 ceph-mon[82531]: pgmap v67: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:57:31.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:31 smithi043 ceph-mon[82531]: osdmap e109: 8 total, 7 up, 8 in 2024-05-15T15:57:31.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:31 smithi043 ceph-mon[88099]: pgmap v67: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:57:31.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:31 smithi043 ceph-mon[88099]: osdmap e109: 8 total, 7 up, 8 in 2024-05-15T15:57:31.337 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:31 smithi043 podman[95604]: 2024-05-15 15:57:31.08350656 +0000 UTC m=+0.673367560 container died bfb528c26072bc42753ff7e23649bc34332c78ecaf396acb5a80eb459c7b31f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True) 2024-05-15T15:57:31.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:31 smithi053 ceph-mon[67220]: pgmap v67: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:57:31.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:31 smithi053 ceph-mon[67220]: osdmap e109: 8 total, 7 up, 8 in 2024-05-15T15:57:32.086 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:31 smithi043 podman[95631]: 2024-05-15 15:57:31.670708584 +0000 UTC m=+0.578722581 container remove bfb528c26072bc42753ff7e23649bc34332c78ecaf396acb5a80eb459c7b31f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-05-15T15:57:32.086 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:31 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.0.service: Deactivated successfully. 2024-05-15T15:57:32.086 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:31 smithi043 systemd[1]: Stopped Ceph osd.0 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:57:32.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:31 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.0.service: Consumed 15.457s CPU time. 2024-05-15T15:57:32.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:31 smithi043 systemd[1]: Starting Ceph osd.0 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:57:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:32 smithi053 ceph-mon[67220]: Health check failed: Degraded data redundancy: 67/627 objects degraded (10.686%), 20 pgs degraded (PG_DEGRADED) 2024-05-15T15:57:32.430 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:32 smithi043 ceph-mon[82531]: Health check failed: Degraded data redundancy: 67/627 objects degraded (10.686%), 20 pgs degraded (PG_DEGRADED) 2024-05-15T15:57:32.431 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:32 smithi043 ceph-mon[88099]: Health check failed: Degraded data redundancy: 67/627 objects degraded (10.686%), 20 pgs degraded (PG_DEGRADED) 2024-05-15T15:57:32.431 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 podman[95743]: 2024-05-15 15:57:32.019986191 +0000 UTC m=+0.017837717 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:57:32.431 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 podman[95743]: 2024-05-15 15:57:32.147066448 +0000 UTC m=+0.144917967 container create 4ff22be7978305701965d49e179809da298595e337b3c9eb789182127931656e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-05-15T15:57:32.431 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 podman[95743]: 2024-05-15 15:57:32.32486093 +0000 UTC m=+0.322712469 container init 4ff22be7978305701965d49e179809da298595e337b3c9eb789182127931656e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-15T15:57:32.431 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 podman[95743]: 2024-05-15 15:57:32.328918046 +0000 UTC m=+0.326769581 container start 4ff22be7978305701965d49e179809da298595e337b3c9eb789182127931656e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-05-15T15:57:32.431 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 podman[95743]: 2024-05-15 15:57:32.429432719 +0000 UTC m=+0.427284257 container attach 4ff22be7978305701965d49e179809da298595e337b3c9eb789182127931656e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T15:57:33.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate[95758]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:57:33.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 bash[95743]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:57:33.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 bash[95743]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:57:33.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 bash[95743]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-15T15:57:33.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 bash[95743]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-15T15:57:33.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 bash[95743]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-15T15:57:33.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 bash[95743]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-15T15:57:33.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 bash[95743]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-05-15T15:57:33.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 bash[95743]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-15T15:57:33.087 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 bash[95743]: --> ceph-volume raw activate successful for osd ID: 0 2024-05-15T15:57:33.088 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate[95758]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:57:33.088 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate[95758]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-15T15:57:33.088 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate[95758]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-15T15:57:33.088 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate[95758]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-15T15:57:33.088 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate[95758]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-15T15:57:33.088 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate[95758]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-05-15T15:57:33.088 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate[95758]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-15T15:57:33.088 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate[95758]: --> ceph-volume raw activate successful for osd ID: 0 2024-05-15T15:57:33.088 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:32 smithi043 podman[95743]: 2024-05-15 15:57:32.853425764 +0000 UTC m=+0.851277290 container died 4ff22be7978305701965d49e179809da298595e337b3c9eb789182127931656e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True) 2024-05-15T15:57:33.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:33 smithi053 ceph-mon[67220]: pgmap v69: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:33.488 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:33 smithi043 ceph-mon[82531]: pgmap v69: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:33.488 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:33 smithi043 ceph-mon[88099]: pgmap v69: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:33.818 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:33 smithi043 podman[95897]: 2024-05-15 15:57:33.486918694 +0000 UTC m=+0.623000125 container remove 4ff22be7978305701965d49e179809da298595e337b3c9eb789182127931656e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240514) 2024-05-15T15:57:33.818 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:33 smithi043 podman[95956]: 2024-05-15 15:57:33.607410806 +0000 UTC m=+0.017970729 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:57:33.818 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:33 smithi043 podman[95956]: 2024-05-15 15:57:33.712850763 +0000 UTC m=+0.123410677 container create d646f037d33caa495ddcf065774f1d2982f990746d26bf9dae252f87b31889e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-05-15T15:57:34.336 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:33 smithi043 podman[95956]: 2024-05-15 15:57:33.932382693 +0000 UTC m=+0.342942601 container init d646f037d33caa495ddcf065774f1d2982f990746d26bf9dae252f87b31889e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T15:57:34.336 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:33 smithi043 podman[95956]: 2024-05-15 15:57:33.935939585 +0000 UTC m=+0.346499501 container start d646f037d33caa495ddcf065774f1d2982f990746d26bf9dae252f87b31889e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, io.buildah.version=1.33.5) 2024-05-15T15:57:34.336 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:33 smithi043 ceph-osd[95974]: -- 172.21.15.43:0/1614460789 <== mon.0 v2:172.21.15.43:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56452054b680 con 0x564520514c00 2024-05-15T15:57:34.336 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:34 smithi043 bash[95956]: d646f037d33caa495ddcf065774f1d2982f990746d26bf9dae252f87b31889e5 2024-05-15T15:57:34.336 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:34 smithi043 systemd[1]: Started Ceph osd.0 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:57:34.836 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:34 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[95970]: 2024-05-15T15:57:34.478+0000 7f3ce78dc7c0 -1 Falling back to public interface 2024-05-15T15:57:35.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[82531]: pgmap v70: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[88099]: pgmap v70: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:35.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:35 smithi053 ceph-mon[67220]: pgmap v70: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:35.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:35.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:36.666 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:57:36] "GET /metrics HTTP/1.1" 200 38140 "" "Prometheus/2.43.0" 2024-05-15T15:57:37.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:36 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:37.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:36 smithi043 ceph-mon[82531]: pgmap v71: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:37.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:36 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:37.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:36 smithi043 ceph-mon[88099]: pgmap v71: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:37.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:36 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:37.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:36 smithi053 ceph-mon[67220]: pgmap v71: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:37.472 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[95970]: 2024-05-15T15:57:37.217+0000 7f3ce78dc7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-05-15T15:57:37.472 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[95970]: 2024-05-15T15:57:37.244+0000 7f3ce78dc7c0 -1 osd.0 107 log_to_monitors true 2024-05-15T15:57:38.086 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 15:57:37 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[95970]: 2024-05-15T15:57:37.860+0000 7f3cd8d33700 -1 osd.0 107 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:57:38.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:37 smithi043 ceph-mon[82531]: from='osd.0 [v2:172.21.15.43:6802/1810639363,v1:172.21.15.43:6803/1810639363]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:57:38.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:37 smithi043 ceph-mon[82531]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:57:38.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:37 smithi043 ceph-mon[88099]: from='osd.0 [v2:172.21.15.43:6802/1810639363,v1:172.21.15.43:6803/1810639363]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:57:38.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:37 smithi043 ceph-mon[88099]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:57:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:37 smithi053 ceph-mon[67220]: from='osd.0 [v2:172.21.15.43:6802/1810639363,v1:172.21.15.43:6803/1810639363]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:57:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:37 smithi053 ceph-mon[67220]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-15T15:57:39.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[88099]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-15T15:57:39.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[88099]: osdmap e110: 8 total, 7 up, 8 in 2024-05-15T15:57:39.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[88099]: from='osd.0 [v2:172.21.15.43:6802/1810639363,v1:172.21.15.43:6803/1810639363]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:57:39.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[88099]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:57:39.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[88099]: pgmap v73: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:39.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:39.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:39.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[82531]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-15T15:57:39.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[82531]: osdmap e110: 8 total, 7 up, 8 in 2024-05-15T15:57:39.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[82531]: from='osd.0 [v2:172.21.15.43:6802/1810639363,v1:172.21.15.43:6803/1810639363]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:57:39.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[82531]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:57:39.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[82531]: pgmap v73: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:39.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:39.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:38 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:39.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:38 smithi053 ceph-mon[67220]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-15T15:57:39.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:38 smithi053 ceph-mon[67220]: osdmap e110: 8 total, 7 up, 8 in 2024-05-15T15:57:39.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:38 smithi053 ceph-mon[67220]: from='osd.0 [v2:172.21.15.43:6802/1810639363,v1:172.21.15.43:6803/1810639363]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:57:39.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:38 smithi053 ceph-mon[67220]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:57:39.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:38 smithi053 ceph-mon[67220]: pgmap v73: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 67/627 objects degraded (10.686%) 2024-05-15T15:57:39.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:38 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:39.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:38 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[82531]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T15:57:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:57:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[82531]: osd.0 [v2:172.21.15.43:6802/1810639363,v1:172.21.15.43:6803/1810639363] boot 2024-05-15T15:57:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[82531]: osdmap e111: 8 total, 8 up, 8 in 2024-05-15T15:57:40.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:40.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[88099]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T15:57:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:57:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[88099]: osd.0 [v2:172.21.15.43:6802/1810639363,v1:172.21.15.43:6803/1810639363] boot 2024-05-15T15:57:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[88099]: osdmap e111: 8 total, 8 up, 8 in 2024-05-15T15:57:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:40.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:39 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:40.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:39 smithi053 ceph-mon[67220]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T15:57:40.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:39 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-15T15:57:40.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:39 smithi053 ceph-mon[67220]: osd.0 [v2:172.21.15.43:6802/1810639363,v1:172.21.15.43:6803/1810639363] boot 2024-05-15T15:57:40.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:39 smithi053 ceph-mon[67220]: osdmap e111: 8 total, 8 up, 8 in 2024-05-15T15:57:40.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:39 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:40.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:39 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:41.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:40 smithi043 ceph-mon[82531]: osdmap e112: 8 total, 8 up, 8 in 2024-05-15T15:57:41.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:40 smithi043 ceph-mon[82531]: pgmap v76: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-05-15T15:57:41.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:40 smithi043 ceph-mon[88099]: osdmap e112: 8 total, 8 up, 8 in 2024-05-15T15:57:41.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:40 smithi043 ceph-mon[88099]: pgmap v76: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-05-15T15:57:41.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:40 smithi053 ceph-mon[67220]: osdmap e112: 8 total, 8 up, 8 in 2024-05-15T15:57:41.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:40 smithi053 ceph-mon[67220]: pgmap v76: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 67/627 objects degraded (10.686%) 2024-05-15T15:57:42.323 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:42 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded (PG_DEGRADED) 2024-05-15T15:57:42.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:42 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded (PG_DEGRADED) 2024-05-15T15:57:42.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:42 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded (PG_DEGRADED) 2024-05-15T15:57:43.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[88099]: pgmap v77: 161 pgs: 8 active+undersized+degraded, 13 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 29/627 objects degraded (4.625%) 2024-05-15T15:57:43.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:43.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:43.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:43.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:57:43.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:43.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[82531]: pgmap v77: 161 pgs: 8 active+undersized+degraded, 13 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 29/627 objects degraded (4.625%) 2024-05-15T15:57:43.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:43.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:43.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:43.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:57:43.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:43 smithi053 ceph-mon[67220]: pgmap v77: 161 pgs: 8 active+undersized+degraded, 13 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 29/627 objects degraded (4.625%) 2024-05-15T15:57:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:43.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:57:43.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:44.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[82531]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[82531]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[82531]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded) 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[88099]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[88099]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:57:44.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:44.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:44.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:44.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:57:44.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:44 smithi043 ceph-mon[88099]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded) 2024-05-15T15:57:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:44 smithi053 ceph-mon[67220]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-05-15T15:57:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:44 smithi053 ceph-mon[67220]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s 2024-05-15T15:57:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:44 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:57:44.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:44 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:44.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:44 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:57:44.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:44 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:44.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:44 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:57:44.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:44 smithi053 ceph-mon[67220]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded) 2024-05-15T15:57:45.067 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:57:44.860+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:45.068 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:45.069 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:45.069 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:45.069 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:45.069 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:45.069 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:45.069 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:45.069 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:57:45.070 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:45.071 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:45.072 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:44 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:45.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:45.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:45.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:45.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:45.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-15T15:57:45.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:45.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:45.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:45.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:45.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-15T15:57:45.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:45.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:45.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:45.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:45.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-15T15:57:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:46 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:57:46.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:46.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:46.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:57:46.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:46.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:46.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:46.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:46.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:46 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:46.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:46.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:46.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:46.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:46.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:46.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:46.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:46.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:46.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:57:46.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:57:46.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:57:46.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:57:46.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:57:46.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:57:46.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:46.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:46.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:46.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:46.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:46.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:57:46.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:46.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:46.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:46.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:57:46.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:46.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:46.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:46.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:46.347 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:46.347 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:46.347 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:46.347 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.347 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:46 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-15T15:57:46.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:46 smithi043 ceph-mon[82531]: Upgrade: osd.1 is safe to restart 2024-05-15T15:57:46.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:46 smithi043 ceph-mon[82531]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 828 B/s rd, 0 op/s 2024-05-15T15:57:46.347 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:46.347 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:46.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:46.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.348 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.348 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:46 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-15T15:57:46.348 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:46 smithi043 ceph-mon[88099]: Upgrade: osd.1 is safe to restart 2024-05-15T15:57:46.348 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:46 smithi043 ceph-mon[88099]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 828 B/s rd, 0 op/s 2024-05-15T15:57:46.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:46 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:46.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:46.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:57:46.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:46 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-15T15:57:46.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:46 smithi053 ceph-mon[67220]: Upgrade: osd.1 is safe to restart 2024-05-15T15:57:46.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:46 smithi053 ceph-mon[67220]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 828 B/s rd, 0 op/s 2024-05-15T15:57:46.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:57:46] "GET /metrics HTTP/1.1" 200 38140 "" "Prometheus/2.43.0" 2024-05-15T15:57:47.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:47 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:47.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:47 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:47.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:47 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-15T15:57:47.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:47 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:47.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:47 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:47.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:47 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:47.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:47 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-15T15:57:47.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:47 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:47.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:47 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:47.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:47 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:47.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:47 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-15T15:57:47.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:47 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:57:48.241 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:48 smithi043 ceph-mon[82531]: Upgrade: Updating osd.1 2024-05-15T15:57:48.242 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:48 smithi043 ceph-mon[82531]: Deploying daemon osd.1 on smithi043 2024-05-15T15:57:48.242 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:48 smithi043 ceph-mon[82531]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T15:57:48.242 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:48 smithi043 ceph-mon[88099]: Upgrade: Updating osd.1 2024-05-15T15:57:48.242 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:48 smithi043 ceph-mon[88099]: Deploying daemon osd.1 on smithi043 2024-05-15T15:57:48.242 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:48 smithi043 ceph-mon[88099]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T15:57:48.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:48 smithi053 ceph-mon[67220]: Upgrade: Updating osd.1 2024-05-15T15:57:48.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:48 smithi053 ceph-mon[67220]: Deploying daemon osd.1 on smithi043 2024-05-15T15:57:48.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:48 smithi053 ceph-mon[67220]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T15:57:50.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:49 smithi043 systemd[1]: Stopping Ceph osd.1 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:57:50.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:50 smithi043 ceph-mon[82531]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:57:50.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:50 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:50.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:50 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:50.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:50 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:50.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:50 smithi043 ceph-mon[82531]: osd.1 marked itself down and dead 2024-05-15T15:57:50.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:50 smithi043 ceph-mon[88099]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:57:50.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:50 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:50.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:50 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:50.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:50 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:50.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:50 smithi043 ceph-mon[88099]: osd.1 marked itself down and dead 2024-05-15T15:57:50.337 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:50 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[34927]: 2024-05-15T15:57:50.020+0000 7fea627a7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T15:57:50.337 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:50 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[34927]: 2024-05-15T15:57:50.020+0000 7fea627a7700 -1 osd.1 112 *** Got signal Terminated *** 2024-05-15T15:57:50.337 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:50 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[34927]: 2024-05-15T15:57:50.020+0000 7fea627a7700 -1 osd.1 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T15:57:50.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:50 smithi053 ceph-mon[67220]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T15:57:50.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:50 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:50.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:50 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:57:50.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:50 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:50.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:50 smithi053 ceph-mon[67220]: osd.1 marked itself down and dead 2024-05-15T15:57:51.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:51 smithi043 ceph-mon[82531]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:57:51.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:51 smithi043 ceph-mon[82531]: osdmap e113: 8 total, 7 up, 8 in 2024-05-15T15:57:51.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:51 smithi043 ceph-mon[88099]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:57:51.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:51 smithi043 ceph-mon[88099]: osdmap e113: 8 total, 7 up, 8 in 2024-05-15T15:57:51.337 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:51 smithi043 podman[100214]: 2024-05-15 15:57:51.154908529 +0000 UTC m=+1.276695623 container died 73d8c7bbd846c04f025204e911c0dc09b8b7267a6c67e7fa834691a8447dac09 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1, RELEASE=HEAD, release=754, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, version=8, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git) 2024-05-15T15:57:51.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:51 smithi053 ceph-mon[67220]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:57:51.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:51 smithi053 ceph-mon[67220]: osdmap e113: 8 total, 7 up, 8 in 2024-05-15T15:57:51.633 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:51 smithi043 podman[100214]: 2024-05-15 15:57:51.620160027 +0000 UTC m=+1.741947135 container cleanup 73d8c7bbd846c04f025204e911c0dc09b8b7267a6c67e7fa834691a8447dac09 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1, com.redhat.component=centos-stream-container, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., distribution-scope=public, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, version=8, GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD) 2024-05-15T15:57:51.633 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:51 smithi043 bash[100214]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1 2024-05-15T15:57:51.730 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:57:52.075 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:51 smithi043 podman[100231]: 2024-05-15 15:57:51.930386564 +0000 UTC m=+0.767892571 container remove 73d8c7bbd846c04f025204e911c0dc09b8b7267a6c67e7fa834691a8447dac09 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, distribution-scope=public, vcs-type=git, build-date=2022-05-03T08:36:31.336870, release=754, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_BRANCH=HEAD) 2024-05-15T15:57:52.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:52 smithi043 ceph-mon[88099]: pgmap v84: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:52.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:52 smithi043 ceph-mon[88099]: osdmap e114: 8 total, 7 up, 8 in 2024-05-15T15:57:52.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:52 smithi043 ceph-mon[82531]: pgmap v84: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:52.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:52 smithi043 ceph-mon[82531]: osdmap e114: 8 total, 7 up, 8 in 2024-05-15T15:57:52.337 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:52 smithi043 podman[100374]: 2024-05-15 15:57:52.186748331 +0000 UTC m=+0.025155889 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:57:52.337 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:52 smithi043 podman[100374]: 2024-05-15 15:57:52.306319501 +0000 UTC m=+0.144727038 container create 9c9522f30cecce5c30b4d1d5214a1d19197a7116c03488fb194cb765e324dfa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514) 2024-05-15T15:57:52.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:52 smithi053 ceph-mon[67220]: pgmap v84: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:57:52.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:52 smithi053 ceph-mon[67220]: osdmap e114: 8 total, 7 up, 8 in 2024-05-15T15:57:52.476 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:57:52.476 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (4m) 14s ago 27m 27.1M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:57:52.476 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (4m) 2m ago 26m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:57:52.476 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 14s ago 26m 64.9M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:57:52.476 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (6m) 2m ago 31m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:57:52.476 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (16m) 14s ago 33m 506M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:57:52.476 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (2m) 14s ago 33m 45.3M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T15:57:52.476 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (3m) 2m ago 31m 38.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:57:52.476 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (2m) 14s ago 31m 37.1M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (5m) 14s ago 27m 15.6M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (5m) 2m ago 27m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (18s) 14s ago 31m 12.6M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (30m) 14s ago 30m 59.0M 2993M 17.2.0 e1d6a67b021e 73d8c7bbd846 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (30m) 14s ago 30m 51.4M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (30m) 14s ago 30m 54.6M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (29m) 2m ago 29m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (29m) 2m ago 29m 51.9M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (28m) 2m ago 28m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (28m) 2m ago 28m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (5m) 2m ago 27m 68.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (26m) 14s ago 26m 77.9M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:57:52.477 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (26m) 2m ago 26m 76.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:57:52.677 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:52 smithi043 podman[100374]: 2024-05-15 15:57:52.560340053 +0000 UTC m=+0.398747619 container init 9c9522f30cecce5c30b4d1d5214a1d19197a7116c03488fb194cb765e324dfa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.vendor=CentOS) 2024-05-15T15:57:52.677 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:52 smithi043 podman[100374]: 2024-05-15 15:57:52.56570992 +0000 UTC m=+0.404117467 container start 9c9522f30cecce5c30b4d1d5214a1d19197a7116c03488fb194cb765e324dfa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-15T15:57:52.677 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:52 smithi043 podman[100374]: 2024-05-15 15:57:52.675778768 +0000 UTC m=+0.514186340 container attach 9c9522f30cecce5c30b4d1d5214a1d19197a7116c03488fb194cb765e324dfa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T15:57:52.900 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:57:52.900 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:57:52.900 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T15:57:52.900 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:57:52.900 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 1 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 6 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:57:52.901 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:57:53.076 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:52 smithi043 podman[100374]: 2024-05-15 15:57:52.892001336 +0000 UTC m=+0.730408893 container died 9c9522f30cecce5c30b4d1d5214a1d19197a7116c03488fb194cb765e324dfa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T15:57:53.258 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:57:53.258 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T15:57:53.259 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:57:53.259 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-05-15T15:57:53.259 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:57:53.259 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "1/8 daemons upgraded", 2024-05-15T15:57:53.259 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading osd daemons", 2024-05-15T15:57:53.259 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:57:53.259 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:57:53.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:53 smithi043 ceph-mon[82531]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:53.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:53 smithi043 ceph-mon[82531]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:53.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:53 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/158382917' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:53.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:53 smithi043 ceph-mon[88099]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:53.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:53 smithi043 ceph-mon[88099]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:53.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:53 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/158382917' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:53.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:53 smithi053 ceph-mon[67220]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:53.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:53 smithi053 ceph-mon[67220]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:53.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:53 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/158382917' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:57:53.836 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:53 smithi043 podman[100443]: 2024-05-15 15:57:53.480051426 +0000 UTC m=+0.582004286 container remove 9c9522f30cecce5c30b4d1d5214a1d19197a7116c03488fb194cb765e324dfa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T15:57:53.836 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:53 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.1.service: Deactivated successfully. 2024-05-15T15:57:53.836 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:53 smithi043 systemd[1]: Stopped Ceph osd.1 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:57:53.837 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:53 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.1.service: Consumed 16.095s CPU time. 2024-05-15T15:57:53.837 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:53 smithi043 systemd[1]: Starting Ceph osd.1 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:57:54.202 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:54 smithi043 ceph-mon[82531]: from='client.54229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:54.202 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:54 smithi043 ceph-mon[82531]: pgmap v86: 161 pgs: 1 active+undersized+degraded, 19 stale+active+clean, 6 active+undersized, 135 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 6/627 objects degraded (0.957%) 2024-05-15T15:57:54.202 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:54 smithi043 ceph-mon[82531]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-05-15T15:57:54.202 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:54 smithi043 ceph-mon[82531]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:54.202 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:54 smithi043 ceph-mon[88099]: from='client.54229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:54.202 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:54 smithi043 ceph-mon[88099]: pgmap v86: 161 pgs: 1 active+undersized+degraded, 19 stale+active+clean, 6 active+undersized, 135 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 6/627 objects degraded (0.957%) 2024-05-15T15:57:54.202 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:54 smithi043 ceph-mon[88099]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-05-15T15:57:54.202 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:54 smithi043 ceph-mon[88099]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:54.203 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:53 smithi043 podman[100569]: 2024-05-15 15:57:53.939588391 +0000 UTC m=+0.109914693 container create 3ca28a47275e07ee8d8cff227e1d348481478dcd7f9647c7dd9f6d66bf3076bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5) 2024-05-15T15:57:54.203 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:53 smithi043 podman[100569]: 2024-05-15 15:57:53.848516518 +0000 UTC m=+0.018842800 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:57:54.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:54 smithi053 ceph-mon[67220]: from='client.54229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:54.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:54 smithi053 ceph-mon[67220]: pgmap v86: 161 pgs: 1 active+undersized+degraded, 19 stale+active+clean, 6 active+undersized, 135 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 6/627 objects degraded (0.957%) 2024-05-15T15:57:54.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:54 smithi053 ceph-mon[67220]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-05-15T15:57:54.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:54 smithi053 ceph-mon[67220]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:57:54.586 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 podman[100569]: 2024-05-15 15:57:54.200483891 +0000 UTC m=+0.370810143 container init 3ca28a47275e07ee8d8cff227e1d348481478dcd7f9647c7dd9f6d66bf3076bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-65480-squid-de5210d, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-05-15T15:57:54.586 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 podman[100569]: 2024-05-15 15:57:54.204492204 +0000 UTC m=+0.374818461 container start 3ca28a47275e07ee8d8cff227e1d348481478dcd7f9647c7dd9f6d66bf3076bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T15:57:54.586 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 podman[100569]: 2024-05-15 15:57:54.325462176 +0000 UTC m=+0.495788448 container attach 3ca28a47275e07ee8d8cff227e1d348481478dcd7f9647c7dd9f6d66bf3076bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-05-15T15:57:55.022 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate[100584]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate[100584]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 bash[100569]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 bash[100569]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 bash[100569]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 bash[100569]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 bash[100569]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 bash[100569]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 bash[100569]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 bash[100569]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 bash[100569]: --> ceph-volume raw activate successful for osd ID: 1 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate[100584]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate[100584]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate[100584]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-15T15:57:55.023 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate[100584]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-15T15:57:55.024 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate[100584]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-05-15T15:57:55.024 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate[100584]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-15T15:57:55.024 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate[100584]: --> ceph-volume raw activate successful for osd ID: 1 2024-05-15T15:57:55.024 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:54 smithi043 podman[100569]: 2024-05-15 15:57:54.728076017 +0000 UTC m=+0.898402276 container died 3ca28a47275e07ee8d8cff227e1d348481478dcd7f9647c7dd9f6d66bf3076bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-05-15T15:57:55.586 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:55 smithi043 podman[100707]: 2024-05-15 15:57:55.287694725 +0000 UTC m=+0.552768757 container remove 3ca28a47275e07ee8d8cff227e1d348481478dcd7f9647c7dd9f6d66bf3076bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-05-15T15:57:55.586 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:55 smithi043 podman[100781]: 2024-05-15 15:57:55.496985499 +0000 UTC m=+0.107239493 container create 9bdd94dfda49a86787d198dec23f2c86c7f23ed00bbf5c1f150b9b82c4c917ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-05-15T15:57:55.586 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:55 smithi043 podman[100781]: 2024-05-15 15:57:55.407605334 +0000 UTC m=+0.017859345 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:57:56.080 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:55 smithi043 podman[100781]: 2024-05-15 15:57:55.740953952 +0000 UTC m=+0.351207948 container init 9bdd94dfda49a86787d198dec23f2c86c7f23ed00bbf5c1f150b9b82c4c917ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-05-15T15:57:56.081 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:55 smithi043 podman[100781]: 2024-05-15 15:57:55.744380553 +0000 UTC m=+0.354634563 container start 9bdd94dfda49a86787d198dec23f2c86c7f23ed00bbf5c1f150b9b82c4c917ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514) 2024-05-15T15:57:56.081 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:55 smithi043 ceph-osd[100799]: -- 172.21.15.43:0/1433538647 <== mon.1 v2:172.21.15.43:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557c3fc55680 con 0x557c3fc1f000 2024-05-15T15:57:56.081 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:55 smithi043 bash[100781]: 9bdd94dfda49a86787d198dec23f2c86c7f23ed00bbf5c1f150b9b82c4c917ae 2024-05-15T15:57:56.081 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:55 smithi043 systemd[1]: Started Ceph osd.1 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:57:56.336 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[100795]: 2024-05-15T15:57:56.285+0000 7f020ba6a7c0 -1 Falling back to public interface 2024-05-15T15:57:56.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:56 smithi043 ceph-mon[82531]: pgmap v87: 161 pgs: 1 active+undersized+degraded, 19 stale+active+clean, 6 active+undersized, 135 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 6/627 objects degraded (0.957%) 2024-05-15T15:57:56.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:56 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:56.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:56 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:56.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:56 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:56.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:56 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:56.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:56 smithi043 ceph-mon[88099]: pgmap v87: 161 pgs: 1 active+undersized+degraded, 19 stale+active+clean, 6 active+undersized, 135 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 6/627 objects degraded (0.957%) 2024-05-15T15:57:56.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:56 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:56.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:56 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:56.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:56 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:56.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:56 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:56.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:56 smithi053 ceph-mon[67220]: pgmap v87: 161 pgs: 1 active+undersized+degraded, 19 stale+active+clean, 6 active+undersized, 135 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 6/627 objects degraded (0.957%) 2024-05-15T15:57:56.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:56 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:56.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:56 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:56.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:56 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:57:56.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:56 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:57:56.799 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:57:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:57:56] "GET /metrics HTTP/1.1" 200 38155 "" "Prometheus/2.43.0" 2024-05-15T15:57:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:57 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:57 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:57 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:57:58.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:58 smithi053 ceph-mon[67220]: pgmap v88: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T15:57:58.385 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:58 smithi043 ceph-mon[82531]: pgmap v88: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T15:57:58.385 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:58 smithi043 ceph-mon[88099]: pgmap v88: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T15:57:59.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:57:59 smithi043 ceph-mon[88099]: from='osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-15T15:57:59.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:57:59 smithi043 ceph-mon[82531]: from='osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-15T15:57:59.337 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[100795]: 2024-05-15T15:57:59.011+0000 7f020ba6a7c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-05-15T15:57:59.337 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[100795]: 2024-05-15T15:57:59.038+0000 7f020ba6a7c0 -1 osd.1 112 log_to_monitors true 2024-05-15T15:57:59.337 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 15:57:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[100795]: 2024-05-15T15:57:59.120+0000 7f01fcec1700 -1 osd.1 112 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:57:59.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:57:59 smithi053 ceph-mon[67220]: from='osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-15T15:58:00.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:00 smithi043 ceph-mon[88099]: pgmap v89: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T15:58:00.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:00 smithi043 ceph-mon[88099]: from='osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-15T15:58:00.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:00 smithi043 ceph-mon[88099]: osdmap e115: 8 total, 7 up, 8 in 2024-05-15T15:58:00.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:00 smithi043 ceph-mon[88099]: from='osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:58:00.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:00 smithi043 ceph-mon[82531]: pgmap v89: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T15:58:00.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:00 smithi043 ceph-mon[82531]: from='osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-15T15:58:00.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:00 smithi043 ceph-mon[82531]: osdmap e115: 8 total, 7 up, 8 in 2024-05-15T15:58:00.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:00 smithi043 ceph-mon[82531]: from='osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:58:00.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:00 smithi053 ceph-mon[67220]: pgmap v89: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T15:58:00.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:00 smithi053 ceph-mon[67220]: from='osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-15T15:58:00.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:00 smithi053 ceph-mon[67220]: osdmap e115: 8 total, 7 up, 8 in 2024-05-15T15:58:00.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:00 smithi053 ceph-mon[67220]: from='osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:58:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:01 smithi053 ceph-mon[67220]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T15:58:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:01 smithi053 ceph-mon[67220]: osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185] boot 2024-05-15T15:58:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:01 smithi053 ceph-mon[67220]: osdmap e116: 8 total, 8 up, 8 in 2024-05-15T15:58:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:01 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:58:01.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:01 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:01.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:01 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:01.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:01 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 20 pgs degraded (PG_DEGRADED) 2024-05-15T15:58:01.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:01 smithi053 ceph-mon[67220]: osdmap e117: 8 total, 8 up, 8 in 2024-05-15T15:58:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[82531]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T15:58:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[82531]: osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185] boot 2024-05-15T15:58:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[82531]: osdmap e116: 8 total, 8 up, 8 in 2024-05-15T15:58:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:58:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:01.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 20 pgs degraded (PG_DEGRADED) 2024-05-15T15:58:01.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[82531]: osdmap e117: 8 total, 8 up, 8 in 2024-05-15T15:58:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[88099]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T15:58:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[88099]: osd.1 [v2:172.21.15.43:6810/1528654185,v1:172.21.15.43:6811/1528654185] boot 2024-05-15T15:58:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[88099]: osdmap e116: 8 total, 8 up, 8 in 2024-05-15T15:58:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-15T15:58:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 20 pgs degraded (PG_DEGRADED) 2024-05-15T15:58:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:01 smithi043 ceph-mon[88099]: osdmap e117: 8 total, 8 up, 8 in 2024-05-15T15:58:02.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:02 smithi053 ceph-mon[67220]: pgmap v92: 161 pgs: 37 peering, 7 active+undersized+degraded, 19 active+undersized, 98 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 35/627 objects degraded (5.582%) 2024-05-15T15:58:02.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:02 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:02.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:02 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:02.388 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:02 smithi043 ceph-mon[88099]: pgmap v92: 161 pgs: 37 peering, 7 active+undersized+degraded, 19 active+undersized, 98 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 35/627 objects degraded (5.582%) 2024-05-15T15:58:02.388 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:02 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:02.388 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:02 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:02.388 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:02 smithi043 ceph-mon[82531]: pgmap v92: 161 pgs: 37 peering, 7 active+undersized+degraded, 19 active+undersized, 98 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 35/627 objects degraded (5.582%) 2024-05-15T15:58:02.388 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:02 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:02.388 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:02 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:04.376 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:04 smithi043 ceph-mon[82531]: pgmap v94: 161 pgs: 37 peering, 6 active+undersized+degraded, 14 active+undersized, 104 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 29/627 objects degraded (4.625%) 2024-05-15T15:58:04.376 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:04 smithi043 ceph-mon[88099]: pgmap v94: 161 pgs: 37 peering, 6 active+undersized+degraded, 14 active+undersized, 104 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 29/627 objects degraded (4.625%) 2024-05-15T15:58:04.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:04 smithi053 ceph-mon[67220]: pgmap v94: 161 pgs: 37 peering, 6 active+undersized+degraded, 14 active+undersized, 104 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 29/627 objects degraded (4.625%) 2024-05-15T15:58:05.877 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:05.877 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:05.877 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:05.877 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: pgmap v95: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:05.877 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:05.877 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:05.877 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:05.878 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:05.878 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: pgmap v96: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 856 B/s rd, 0 op/s 2024-05-15T15:58:05.878 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: pgmap v97: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:58:05.878 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:05.878 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:05.878 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:05.878 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:05.878 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:05.878 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: pgmap v95: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: pgmap v96: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 856 B/s rd, 0 op/s 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: pgmap v97: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:58:05.879 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:05.880 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:05.880 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:05.880 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:05.880 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:05.880 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:05 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:06.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: pgmap v95: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: pgmap v96: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 856 B/s rd, 0 op/s 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: pgmap v97: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 131 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:06.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:06.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:06.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:05 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:06.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:58:06] "GET /metrics HTTP/1.1" 200 38167 "" "Prometheus/2.43.0" 2024-05-15T15:58:06.928 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:06 smithi043 ceph-mon[82531]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded) 2024-05-15T15:58:06.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:06 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:06.929 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:06 smithi043 ceph-mon[88099]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded) 2024-05-15T15:58:06.929 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:06 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:06 smithi053 ceph-mon[67220]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded) 2024-05-15T15:58:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:06 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:07.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:58:06.926+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:07.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:58:07.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:07.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:07.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:07.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:07.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:07.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:58:07.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:07.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:58:07.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:07.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:07.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:07.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:07.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:07.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:07.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:07.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:07.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:07.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:07.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:07.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:07.342 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:07.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:07.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:07.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:58:07.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:07.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:58:07.343 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:07.344 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:58:07.344 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:07.344 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:58:07.344 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:07.344 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:07.344 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:07.345 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:58:07.345 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:07.345 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:07.345 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:07.346 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.346 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.346 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:07.346 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:07.346 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:58:07.346 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:07.347 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:58:07.347 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:07.347 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:07.347 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:07.347 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:07.348 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:07.348 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:07.348 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:07.348 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.348 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.920 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:07.920 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:07.920 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:07.920 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:07.920 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:07.920 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.921 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:07.922 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:07.923 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.924 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:07.925 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:07.926 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:07.927 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:07.928 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:07.928 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:07.928 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:07.928 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:07.928 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:07.928 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:07.928 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.928 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.928 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: Upgrade: Finalizing container_image settings 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:58:07.929 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:07.930 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: Upgrade: Finalizing container_image settings 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:58:07.931 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:58:07.932 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.932 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.932 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:58:07.932 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:58:07.932 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:58:07.932 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:58:07.932 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:58:07.932 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:58:07.932 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:58:07.932 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:58:07.933 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.934 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: Upgrade: Complete! 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:07.935 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:58:07.936 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.937 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: Upgrade: Complete! 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:07.938 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-05-15T15:58:07.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-05-15T15:58:07.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:07.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:07.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:07.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:07.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:07.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:07 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:08.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:08.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:08.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:08.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: Upgrade: Finalizing container_image settings 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:58:08.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:58:08.137 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.138 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: Upgrade: Complete! 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 0 op/s 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:08.139 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:08.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:08.140 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:07 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:09.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:08 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:09.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:08 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:09.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:08 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:09.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:08 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:09.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:08 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:09.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:08 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:09.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:08 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:09.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:08 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:09.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:58:08.951+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:58:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:08 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:08 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:08 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:08 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:10.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:10.336 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:10.336 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:10.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:10.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:10.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.340 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:10.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:10.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:10.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:10.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:10.344 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:10.344 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:10.344 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:10.344 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.344 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.344 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[82531]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[82531]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[88099]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-05-15T15:58:10.346 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[88099]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:58:10.346 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:10.346 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:10.346 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:10.346 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:10.346 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:10.346 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:09 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:10.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:09 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:10.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:10.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:10.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:10.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:09 smithi053 ceph-mon[67220]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 860 B/s rd, 0 op/s 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:09 smithi053 ceph-mon[67220]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:09 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:09 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:10.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:09 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:10.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:09 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:10.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:09 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:10.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:09 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:11.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:58:10.918+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:11.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:11.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:11.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:11.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:11.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:11.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:11.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:11.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:11.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:58:11.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:11.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:11.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:11.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:11.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:11.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:11.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:11.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:11.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:12 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:12.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:12.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:12.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:12.340 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:12 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:12.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:12.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:12.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:12.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:12.345 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:12.345 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:12.345 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:12.345 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.345 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.345 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:12 smithi043 ceph-mon[82531]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-15T15:58:12.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:12.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:12.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:12.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.346 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.346 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:12 smithi043 ceph-mon[88099]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:12 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:12.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:12.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:12.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:12.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:12.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:12.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:12.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:12.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:12.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:12.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:12 smithi053 ceph-mon[67220]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-15T15:58:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:14 smithi043 ceph-mon[82531]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:58:14.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:14 smithi043 ceph-mon[88099]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:58:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:14 smithi053 ceph-mon[67220]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T15:58:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:15 smithi043 ceph-mon[82531]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 642 B/s rd, 0 op/s 2024-05-15T15:58:16.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:16.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:15 smithi043 ceph-mon[88099]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 642 B/s rd, 0 op/s 2024-05-15T15:58:16.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:16.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:15 smithi053 ceph-mon[67220]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 642 B/s rd, 0 op/s 2024-05-15T15:58:16.718 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:58:16] "GET /metrics HTTP/1.1" 200 38167 "" "Prometheus/2.43.0" 2024-05-15T15:58:17.032 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:16 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:17.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:16 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:17.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:16 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:18.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:17 smithi043 ceph-mon[82531]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:58:18.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:17 smithi043 ceph-mon[88099]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:58:18.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:17 smithi053 ceph-mon[67220]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T15:58:20.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:20 smithi043 ceph-mon[82531]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:58:20.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:20 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:20.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:20.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:20 smithi043 ceph-mon[88099]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:58:20.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:20 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:20.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:20.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:20 smithi053 ceph-mon[67220]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:58:20.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:20 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:20.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:22.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:22 smithi043 ceph-mon[82531]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:22.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:22 smithi043 ceph-mon[88099]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:22.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:22 smithi053 ceph-mon[67220]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:24.301 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:24 smithi043 ceph-mon[82531]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:24.301 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:24 smithi043 ceph-mon[88099]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:24.332 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T15:58:24.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:24 smithi053 ceph-mon[67220]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:25.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:25 smithi043 ceph-mon[82531]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:25.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:25 smithi043 ceph-mon[88099]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:25.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:25 smithi053 ceph-mon[67220]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:26 smithi043 ceph-mon[82531]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:26.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:26 smithi043 ceph-mon[88099]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:26.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:26 smithi053 ceph-mon[67220]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:26.478 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:58:26.478 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (5m) 26s ago 27m 27.1M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (5m) 2m ago 27m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 26s ago 26m 64.9M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (6m) 2m ago 32m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (16m) 26s ago 33m 508M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (3m) 26s ago 33m 46.9M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (3m) 2m ago 32m 38.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (2m) 26s ago 32m 38.5M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (6m) 26s ago 28m 15.8M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (6m) 2m ago 28m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (52s) 26s ago 31m 43.2M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (31s) 26s ago 31m 12.5M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (31m) 26s ago 31m 51.3M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (30m) 26s ago 30m 55.3M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (30m) 2m ago 30m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:58:26.479 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (29m) 2m ago 29m 51.9M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:58:26.480 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (29m) 2m ago 29m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:58:26.480 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (29m) 2m ago 29m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:58:26.480 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (5m) 2m ago 28m 68.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:58:26.480 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (26m) 26s ago 26m 78.0M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:58:26.480 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (26m) 2m ago 26m 76.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:58:26.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:58:26] "GET /metrics HTTP/1.1" 200 38166 "" "Prometheus/2.43.0" 2024-05-15T15:58:27.074 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-05-15T15:58:27.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:27 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:27.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:27 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:27.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:27 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:28.247 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:28 smithi043 ceph-mon[82531]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:28.247 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:28 smithi043 ceph-mon[82531]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:28.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:28 smithi043 ceph-mon[88099]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:28.247 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:28 smithi043 ceph-mon[88099]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:28.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:28 smithi053 ceph-mon[67220]: from='client.54259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:28.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:28 smithi053 ceph-mon[67220]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:29.052 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:58:29.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:29 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2563532271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:29.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:29 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2563532271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:29 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2563532271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:29.638 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-05-15T15:58:30.274 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:30 smithi043 ceph-mon[82531]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:30.274 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:30 smithi043 ceph-mon[88099]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:30 smithi053 ceph-mon[67220]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:32.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:32 smithi043 ceph-mon[82531]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:32.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:32 smithi043 ceph-mon[88099]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:32 smithi053 ceph-mon[67220]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:33.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:33 smithi043 ceph-mon[88099]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:33.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:33 smithi043 ceph-mon[82531]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:33.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:33 smithi053 ceph-mon[67220]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:33.751 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:58:34.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:34 smithi043 ceph-mon[82531]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:34.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:34 smithi043 ceph-mon[88099]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:34.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:34 smithi053 ceph-mon[67220]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:34.398 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade status' 2024-05-15T15:58:35.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:35.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:36.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:36 smithi043 ceph-mon[82531]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:36.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:36 smithi043 ceph-mon[88099]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:36.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:36 smithi053 ceph-mon[67220]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:36.384 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:58:36.385 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": null, 2024-05-15T15:58:36.385 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": false, 2024-05-15T15:58:36.385 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "", 2024-05-15T15:58:36.385 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:58:36.385 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": null, 2024-05-15T15:58:36.385 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "", 2024-05-15T15:58:36.385 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:58:36.385 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:58:36.665 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:58:36] "GET /metrics HTTP/1.1" 200 38167 "" "Prometheus/2.43.0" 2024-05-15T15:58:37.061 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T15:58:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:37 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:37.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:37 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:37.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:37 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:38.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:38 smithi043 ceph-mon[82531]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:38.319 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:38 smithi043 ceph-mon[82531]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:38.319 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:38 smithi043 ceph-mon[88099]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:38.319 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:38 smithi043 ceph-mon[88099]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:38 smithi053 ceph-mon[67220]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:38 smithi053 ceph-mon[67220]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:39.128 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:58:39.128 INFO:teuthology.orchestra.run.smithi043.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:58:39.128 INFO:teuthology.orchestra.run.smithi043.stdout: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:58:39.862 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-05-15T15:58:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:40 smithi043 ceph-mon[82531]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:40 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2310808941' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:58:40.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:40 smithi043 ceph-mon[88099]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:40.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:40 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2310808941' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:58:40.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:40 smithi053 ceph-mon[67220]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:40.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:40 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2310808941' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:58:42.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:42 smithi043 ceph-mon[82531]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:42.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:42 smithi043 ceph-mon[88099]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:42.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:42 smithi053 ceph-mon[67220]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:43.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:43 smithi043 ceph-mon[82531]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:43.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:43 smithi043 ceph-mon[88099]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:43 smithi053 ceph-mon[67220]: from='client.44352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:44.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:44 smithi043 ceph-mon[82531]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:44.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:44 smithi043 ceph-mon[88099]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:44 smithi053 ceph-mon[67220]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:44.487 INFO:teuthology.orchestra.run.smithi043.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:58:45.106 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-15T15:58:45.712 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:58:45.712 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1007 B/s rd, 0 op/s 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 0 op/s 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:45.713 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1007 B/s rd, 0 op/s 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 0 op/s 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:45.714 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:45.715 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:45.715 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:45 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1007 B/s rd, 0 op/s 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 0 op/s 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:58:45.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:45.882 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:45 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:58:46.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:46 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:46.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:46 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:46.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:58:46] "GET /metrics HTTP/1.1" 200 38167 "" "Prometheus/2.43.0" 2024-05-15T15:58:46.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:46 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:47.654 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:47 smithi043 ceph-mon[82531]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:47.654 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:47 smithi043 ceph-mon[88099]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:47.782 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:47 smithi053 ceph-mon[67220]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:48.142 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:58:48.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:58:47.945+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:48.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:48.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:48.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:48.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:48.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:48.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:48.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:48.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:48.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:47 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.832 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:58:48.832 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (5m) 48s ago 27m 27.1M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:58:48.832 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (5m) 3m ago 27m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 48s ago 27m 64.9M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (7m) 3m ago 32m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (17m) 48s ago 34m 508M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (3m) 48s ago 34m 46.9M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (4m) 3m ago 32m 38.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (2m) 48s ago 32m 38.5M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (6m) 48s ago 28m 15.8M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (6m) 3m ago 28m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (75s) 48s ago 32m 43.2M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (53s) 48s ago 31m 12.5M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (31m) 48s ago 31m 51.3M 2993M 17.2.0 e1d6a67b021e fe0362695bb1 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (30m) 48s ago 30m 55.3M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:58:48.833 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (30m) 3m ago 30m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:58:48.834 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (30m) 3m ago 30m 51.9M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:58:48.834 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (29m) 3m ago 29m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:58:48.834 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (29m) 3m ago 29m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:58:48.834 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (6m) 3m ago 28m 68.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:58:48.834 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (27m) 48s ago 27m 78.0M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:58:48.834 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (27m) 3m ago 27m 76.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:58:48.834 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:48 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:48.834 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:48.835 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:48 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:48.837 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.838 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:48.839 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:48.839 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:48.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:48.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:48.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:48.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:48.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:48.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:58:48.839 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:48.840 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:48.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:48.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:48.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:48.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:48.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:48.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:48.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:48.841 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:58:48.842 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:58:48.843 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:48.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:48.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:48.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:58:48.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:48.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:48.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:48.844 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:48.844 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:48.844 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:48.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:48.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.845 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:48 smithi043 ceph-mon[82531]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:58:48.845 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:48 smithi043 ceph-mon[82531]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:48.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:48.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:48.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:48.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.845 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:48 smithi043 ceph-mon[88099]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:58:48.846 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:48 smithi043 ceph-mon[88099]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:48 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:48.881 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:58:48.882 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:58:48.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:58:48.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:58:48.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:58:48.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:58:48.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:58:48.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:58:48.883 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:58:48.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:58:48.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:58:48.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:58:48.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:58:48.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:58:48.884 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:58:48.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:58:48.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:58:48.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:58:48.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:58:48.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:58:48.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:58:48.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:58:48.885 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:58:48.886 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:48 smithi053 ceph-mon[67220]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:58:48.887 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:48 smithi053 ceph-mon[67220]: from='client.54295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:58:49.249 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:58:49.250 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:58:49.250 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:58:49.250 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:58:49.250 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-05-15T15:58:49.250 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 7 2024-05-15T15:58:49.250 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:58:49.250 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:58:49.602 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:58:49.603 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", 2024-05-15T15:58:49.603 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:58:49.603 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-05-15T15:58:49.603 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:58:49.603 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "", 2024-05-15T15:58:49.603 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 image", 2024-05-15T15:58:49.603 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:58:49.603 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:58:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:49 smithi043 ceph-mon[88099]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:49 smithi043 ceph-mon[88099]: from='client.44370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:49 smithi043 ceph-mon[88099]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 0 op/s 2024-05-15T15:58:49.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:49 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/4076711198' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:49 smithi043 ceph-mon[82531]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:49 smithi043 ceph-mon[82531]: from='client.44370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:49 smithi043 ceph-mon[82531]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 0 op/s 2024-05-15T15:58:49.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:49 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/4076711198' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:49 smithi053 ceph-mon[67220]: from='client.54301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:49 smithi053 ceph-mon[67220]: from='client.44370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:49 smithi053 ceph-mon[67220]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 0 op/s 2024-05-15T15:58:49.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:49 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/4076711198' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:50.803 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:50 smithi043 ceph-mon[82531]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:50.803 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:50 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:50.804 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:50 smithi043 ceph-mon[88099]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:50.804 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:50 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:50 smithi053 ceph-mon[67220]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:58:50.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:50 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:58:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mgr 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mon 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all crash 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: Upgrade: osd.2 is safe to restart 2024-05-15T15:58:52.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[82531]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mgr 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mon 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all crash 2024-05-15T15:58:52.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-15T15:58:52.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-15T15:58:52.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: Upgrade: osd.2 is safe to restart 2024-05-15T15:58:52.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:51 smithi043 ceph-mon[88099]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:52.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mgr 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mon 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all crash 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-15T15:58:52.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: Upgrade: osd.2 is safe to restart 2024-05-15T15:58:52.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:51 smithi053 ceph-mon[67220]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[82531]: Upgrade: Updating osd.2 2024-05-15T15:58:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:53.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-15T15:58:53.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:53.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[82531]: Deploying daemon osd.2 on smithi043 2024-05-15T15:58:53.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[82531]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:53.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[88099]: Upgrade: Updating osd.2 2024-05-15T15:58:53.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:53.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-15T15:58:53.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:53.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[88099]: Deploying daemon osd.2 on smithi043 2024-05-15T15:58:53.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:53 smithi043 ceph-mon[88099]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:53 smithi053 ceph-mon[67220]: Upgrade: Updating osd.2 2024-05-15T15:58:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:53 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:58:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:53 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-15T15:58:53.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:53 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:58:53.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:53 smithi053 ceph-mon[67220]: Deploying daemon osd.2 on smithi043 2024-05-15T15:58:53.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:53 smithi053 ceph-mon[67220]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:58:56.086 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:55 smithi043 systemd[1]: Stopping Ceph osd.2 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:58:56.573 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:56 smithi043 ceph-mon[82531]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:58:56.573 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:56 smithi043 ceph-mon[82531]: osd.2 marked itself down and dead 2024-05-15T15:58:56.574 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:56 smithi043 ceph-mon[88099]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:58:56.574 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:56 smithi043 ceph-mon[88099]: osd.2 marked itself down and dead 2024-05-15T15:58:56.574 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[37838]: 2024-05-15T15:58:56.110+0000 7ff808acd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T15:58:56.574 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[37838]: 2024-05-15T15:58:56.110+0000 7ff808acd700 -1 osd.2 117 *** Got signal Terminated *** 2024-05-15T15:58:56.574 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[37838]: 2024-05-15T15:58:56.110+0000 7ff808acd700 -1 osd.2 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T15:58:56.574 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:58:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:58:56] "GET /metrics HTTP/1.1" 200 38168 "" "Prometheus/2.43.0" 2024-05-15T15:58:56.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:56 smithi053 ceph-mon[67220]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:58:56.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:56 smithi053 ceph-mon[67220]: osd.2 marked itself down and dead 2024-05-15T15:58:56.836 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:56 smithi043 podman[107530]: 2024-05-15 15:58:56.571626855 +0000 UTC m=+0.566908162 container died fe0362695bb18e8a0d45886d9a6f24a2aeb6f02c1d612884635ba091d34cc76d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, version=8, io.openshift.expose-services=, release=754, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vcs-type=git, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-15T15:58:57.196 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:56 smithi043 podman[107530]: 2024-05-15 15:58:56.977391635 +0000 UTC m=+0.972672931 container cleanup fe0362695bb18e8a0d45886d9a6f24a2aeb6f02c1d612884635ba091d34cc76d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2, version=8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, distribution-scope=public, vendor=Red Hat, Inc., vcs-type=git, GIT_CLEAN=True, release=754, io.buildah.version=1.19.8, io.openshift.expose-services=, ceph=True) 2024-05-15T15:58:57.197 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:56 smithi043 bash[107530]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2 2024-05-15T15:58:57.546 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:57 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:57.546 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:57 smithi043 ceph-mon[88099]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:58:57.546 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:57 smithi043 ceph-mon[88099]: osdmap e118: 8 total, 7 up, 8 in 2024-05-15T15:58:57.547 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:57 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:57.547 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:57 smithi043 ceph-mon[82531]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:58:57.547 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:57 smithi043 ceph-mon[82531]: osdmap e118: 8 total, 7 up, 8 in 2024-05-15T15:58:57.547 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:57 smithi043 podman[107546]: 2024-05-15 15:58:57.212061554 +0000 UTC m=+0.635175383 container remove fe0362695bb18e8a0d45886d9a6f24a2aeb6f02c1d612884635ba091d34cc76d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, version=8, ceph=True, release=754, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_CLEAN=True) 2024-05-15T15:58:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:57 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:58:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:57 smithi053 ceph-mon[67220]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:58:57.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:57 smithi053 ceph-mon[67220]: osdmap e118: 8 total, 7 up, 8 in 2024-05-15T15:58:57.817 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:57 smithi043 podman[107635]: 2024-05-15 15:58:57.447827914 +0000 UTC m=+0.018110024 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:58:57.817 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:57 smithi043 podman[107635]: 2024-05-15 15:58:57.713173332 +0000 UTC m=+0.283455443 container create 02b962d335c4300b7e250027f590b0802a89d95a3e8540ff92062a15b3d83f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-05-15T15:58:58.067 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:57 smithi043 podman[107635]: 2024-05-15 15:58:57.932963804 +0000 UTC m=+0.503245923 container init 02b962d335c4300b7e250027f590b0802a89d95a3e8540ff92062a15b3d83f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T15:58:58.067 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:57 smithi043 podman[107635]: 2024-05-15 15:58:57.936953935 +0000 UTC m=+0.507236045 container start 02b962d335c4300b7e250027f590b0802a89d95a3e8540ff92062a15b3d83f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=) 2024-05-15T15:58:58.067 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:58 smithi043 podman[107635]: 2024-05-15 15:58:58.032673312 +0000 UTC m=+0.602955429 container attach 02b962d335c4300b7e250027f590b0802a89d95a3e8540ff92062a15b3d83f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD) 2024-05-15T15:58:58.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:58 smithi043 ceph-mon[82531]: pgmap v128: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:58:58.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:58 smithi043 ceph-mon[82531]: osdmap e119: 8 total, 7 up, 8 in 2024-05-15T15:58:58.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:58 smithi043 ceph-mon[88099]: pgmap v128: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:58:58.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:58 smithi043 ceph-mon[88099]: osdmap e119: 8 total, 7 up, 8 in 2024-05-15T15:58:58.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:58 smithi043 podman[107635]: 2024-05-15 15:58:58.25079799 +0000 UTC m=+0.821080099 container died 02b962d335c4300b7e250027f590b0802a89d95a3e8540ff92062a15b3d83f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True) 2024-05-15T15:58:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:58 smithi053 ceph-mon[67220]: pgmap v128: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:58:58.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:58 smithi053 ceph-mon[67220]: osdmap e119: 8 total, 7 up, 8 in 2024-05-15T15:58:58.880 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:58 smithi043 podman[107677]: 2024-05-15 15:58:58.861665631 +0000 UTC m=+0.602106801 container remove 02b962d335c4300b7e250027f590b0802a89d95a3e8540ff92062a15b3d83f02 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T15:58:59.256 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:58 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.2.service: Deactivated successfully. 2024-05-15T15:58:59.256 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:58 smithi043 systemd[1]: Stopped Ceph osd.2 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:58:59.256 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:58 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.2.service: Consumed 13.545s CPU time. 2024-05-15T15:58:59.256 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:59 smithi043 systemd[1]: Starting Ceph osd.2 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:58:59.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:58:59 smithi043 ceph-mon[82531]: Health check failed: Degraded data redundancy: 24/627 objects degraded (3.828%), 3 pgs degraded (PG_DEGRADED) 2024-05-15T15:58:59.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:58:59 smithi043 ceph-mon[88099]: Health check failed: Degraded data redundancy: 24/627 objects degraded (3.828%), 3 pgs degraded (PG_DEGRADED) 2024-05-15T15:58:59.569 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:59 smithi043 podman[107777]: 2024-05-15 15:58:59.220350911 +0000 UTC m=+0.018547691 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:58:59.569 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:59 smithi043 podman[107777]: 2024-05-15 15:58:59.337781818 +0000 UTC m=+0.135978629 container create 39f515077ea85b3a98ecab983ab8c84f388e7037a003132f59f95e102a6ef747 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-05-15T15:58:59.569 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:59 smithi043 podman[107777]: 2024-05-15 15:58:59.567280413 +0000 UTC m=+0.365477186 container init 39f515077ea85b3a98ecab983ab8c84f388e7037a003132f59f95e102a6ef747 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d) 2024-05-15T15:58:59.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:58:59 smithi053 ceph-mon[67220]: Health check failed: Degraded data redundancy: 24/627 objects degraded (3.828%), 3 pgs degraded (PG_DEGRADED) 2024-05-15T15:58:59.836 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:59 smithi043 podman[107777]: 2024-05-15 15:58:59.570793614 +0000 UTC m=+0.368990386 container start 39f515077ea85b3a98ecab983ab8c84f388e7037a003132f59f95e102a6ef747 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-05-15T15:58:59.836 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:58:59 smithi043 podman[107777]: 2024-05-15 15:58:59.656919158 +0000 UTC m=+0.455115944 container attach 39f515077ea85b3a98ecab983ab8c84f388e7037a003132f59f95e102a6ef747 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=) 2024-05-15T15:59:00.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:00 smithi043 ceph-mon[82531]: pgmap v130: 161 pgs: 3 active+undersized+degraded, 9 stale+active+clean, 11 active+undersized, 138 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 24/627 objects degraded (3.828%) 2024-05-15T15:59:00.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:00 smithi043 ceph-mon[88099]: pgmap v130: 161 pgs: 3 active+undersized+degraded, 9 stale+active+clean, 11 active+undersized, 138 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 24/627 objects degraded (3.828%) 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate[107791]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate[107791]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 bash[107777]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 bash[107777]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 bash[107777]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 bash[107777]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 bash[107777]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 bash[107777]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 bash[107777]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 bash[107777]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-15T15:59:00.337 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 bash[107777]: --> ceph-volume raw activate successful for osd ID: 2 2024-05-15T15:59:00.338 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate[107791]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-15T15:59:00.338 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate[107791]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-15T15:59:00.338 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate[107791]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-15T15:59:00.338 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate[107791]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-15T15:59:00.338 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate[107791]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-05-15T15:59:00.338 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate[107791]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-15T15:59:00.338 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate[107791]: --> ceph-volume raw activate successful for osd ID: 2 2024-05-15T15:59:00.338 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 podman[107777]: 2024-05-15 15:59:00.091844067 +0000 UTC m=+0.890040835 container died 39f515077ea85b3a98ecab983ab8c84f388e7037a003132f59f95e102a6ef747 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux ) 2024-05-15T15:59:00.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:00 smithi053 ceph-mon[67220]: pgmap v130: 161 pgs: 3 active+undersized+degraded, 9 stale+active+clean, 11 active+undersized, 138 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 24/627 objects degraded (3.828%) 2024-05-15T15:59:00.967 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 podman[107915]: 2024-05-15 15:59:00.644261413 +0000 UTC m=+0.546559465 container remove 39f515077ea85b3a98ecab983ab8c84f388e7037a003132f59f95e102a6ef747 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-05-15T15:59:00.967 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 podman[107975]: 2024-05-15 15:59:00.853792219 +0000 UTC m=+0.108860695 container create 087a14715d561ee617c3aab791bdfd792560e003b403b554080a2245197cab94 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-05-15T15:59:00.967 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:00 smithi043 podman[107975]: 2024-05-15 15:59:00.763476318 +0000 UTC m=+0.018544795 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:59:01.336 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:01 smithi043 podman[107975]: 2024-05-15 15:59:01.072581796 +0000 UTC m=+0.327650273 container init 087a14715d561ee617c3aab791bdfd792560e003b403b554080a2245197cab94 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d) 2024-05-15T15:59:01.336 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:01 smithi043 podman[107975]: 2024-05-15 15:59:01.076053853 +0000 UTC m=+0.331122332 container start 087a14715d561ee617c3aab791bdfd792560e003b403b554080a2245197cab94 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T15:59:01.336 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:01 smithi043 ceph-osd[107994]: -- 172.21.15.43:0/1112089442 <== mon.1 v2:172.21.15.43:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c2c3cb9680 con 0x55c2c3c82c00 2024-05-15T15:59:01.336 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:01 smithi043 bash[107975]: 087a14715d561ee617c3aab791bdfd792560e003b403b554080a2245197cab94 2024-05-15T15:59:01.336 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:01 smithi043 systemd[1]: Started Ceph osd.2 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:59:01.836 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[107990]: 2024-05-15T15:59:01.616+0000 7f68790757c0 -1 Falling back to public interface 2024-05-15T15:59:02.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:02 smithi043 ceph-mon[82531]: pgmap v131: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:02.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:02 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:02.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:02 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:02.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:02 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:02.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:02 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:02 smithi043 ceph-mon[88099]: pgmap v131: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:02 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:02 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:02 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:02 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:02 smithi053 ceph-mon[67220]: pgmap v131: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:02 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:02 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:02.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:02 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:02.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:02 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:04.520 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:04 smithi043 ceph-mon[82531]: pgmap v132: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:04.521 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:04 smithi043 ceph-mon[88099]: pgmap v132: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:04.521 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:04 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[107990]: 2024-05-15T15:59:04.341+0000 7f68790757c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-05-15T15:59:04.521 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:04 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[107990]: 2024-05-15T15:59:04.364+0000 7f68790757c0 -1 osd.2 117 log_to_monitors true 2024-05-15T15:59:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:04 smithi053 ceph-mon[67220]: pgmap v132: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:05.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:05 smithi043 ceph-mon[88099]: from='osd.2 [v2:172.21.15.43:6818/2815006330,v1:172.21.15.43:6819/2815006330]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:59:05.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:05 smithi043 ceph-mon[88099]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:59:05.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:05.543 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:05.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:05 smithi043 ceph-mon[82531]: from='osd.2 [v2:172.21.15.43:6818/2815006330,v1:172.21.15.43:6819/2815006330]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:59:05.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:05 smithi043 ceph-mon[82531]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:59:05.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:05.544 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:05 smithi053 ceph-mon[67220]: from='osd.2 [v2:172.21.15.43:6818/2815006330,v1:172.21.15.43:6819/2815006330]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:59:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:05 smithi053 ceph-mon[67220]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-15T15:59:05.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:05.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:05.836 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 15:59:05 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[107990]: 2024-05-15T15:59:05.540+0000 7f68664c4700 -1 osd.2 117 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T15:59:06.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:59:06] "GET /metrics HTTP/1.1" 200 38166 "" "Prometheus/2.43.0" 2024-05-15T15:59:06.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[82531]: pgmap v133: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:06.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[82531]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[82531]: osdmap e120: 8 total, 7 up, 8 in 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[82531]: from='osd.2 [v2:172.21.15.43:6818/2815006330,v1:172.21.15.43:6819/2815006330]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[82531]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[82531]: from='osd.2 ' entity='osd.2' 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 57/627 objects degraded (9.091%), 15 pgs degraded (PG_DEGRADED) 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[88099]: pgmap v133: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[88099]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[88099]: osdmap e120: 8 total, 7 up, 8 in 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[88099]: from='osd.2 [v2:172.21.15.43:6818/2815006330,v1:172.21.15.43:6819/2815006330]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[88099]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:59:06.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[88099]: from='osd.2 ' entity='osd.2' 2024-05-15T15:59:06.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 57/627 objects degraded (9.091%), 15 pgs degraded (PG_DEGRADED) 2024-05-15T15:59:06.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:06.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:06 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:06 smithi053 ceph-mon[67220]: pgmap v133: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:06.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:06 smithi053 ceph-mon[67220]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-15T15:59:06.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:06 smithi053 ceph-mon[67220]: osdmap e120: 8 total, 7 up, 8 in 2024-05-15T15:59:06.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:06 smithi053 ceph-mon[67220]: from='osd.2 [v2:172.21.15.43:6818/2815006330,v1:172.21.15.43:6819/2815006330]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:59:06.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:06 smithi053 ceph-mon[67220]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T15:59:06.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:06 smithi053 ceph-mon[67220]: from='osd.2 ' entity='osd.2' 2024-05-15T15:59:06.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:06 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 57/627 objects degraded (9.091%), 15 pgs degraded (PG_DEGRADED) 2024-05-15T15:59:06.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:06 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:06.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:06 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:07.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:07.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[88099]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T15:59:07.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:59:07.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[88099]: osd.2 [v2:172.21.15.43:6818/2815006330,v1:172.21.15.43:6819/2815006330] boot 2024-05-15T15:59:07.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[88099]: osdmap e121: 8 total, 8 up, 8 in 2024-05-15T15:59:07.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:07.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:07.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:07.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[82531]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T15:59:07.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:59:07.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[82531]: osd.2 [v2:172.21.15.43:6818/2815006330,v1:172.21.15.43:6819/2815006330] boot 2024-05-15T15:59:07.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[82531]: osdmap e121: 8 total, 8 up, 8 in 2024-05-15T15:59:07.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:07.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:07 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:07 smithi053 ceph-mon[67220]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T15:59:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-15T15:59:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:07 smithi053 ceph-mon[67220]: osd.2 [v2:172.21.15.43:6818/2815006330,v1:172.21.15.43:6819/2815006330] boot 2024-05-15T15:59:07.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:07 smithi053 ceph-mon[67220]: osdmap e121: 8 total, 8 up, 8 in 2024-05-15T15:59:07.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:07.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:08.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:08 smithi043 ceph-mon[82531]: pgmap v136: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:08.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:08 smithi043 ceph-mon[82531]: osdmap e122: 8 total, 8 up, 8 in 2024-05-15T15:59:08.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:08 smithi043 ceph-mon[88099]: pgmap v136: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:08.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:08 smithi043 ceph-mon[88099]: osdmap e122: 8 total, 8 up, 8 in 2024-05-15T15:59:08.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:08 smithi053 ceph-mon[67220]: pgmap v136: 161 pgs: 15 active+undersized+degraded, 32 active+undersized, 114 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-05-15T15:59:08.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:08 smithi053 ceph-mon[67220]: osdmap e122: 8 total, 8 up, 8 in 2024-05-15T15:59:10.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:10 smithi043 ceph-mon[82531]: pgmap v138: 161 pgs: 12 active+undersized+degraded, 25 active+undersized, 124 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 33/627 objects degraded (5.263%) 2024-05-15T15:59:10.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:10 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:10.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:10 smithi043 ceph-mon[88099]: pgmap v138: 161 pgs: 12 active+undersized+degraded, 25 active+undersized, 124 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 33/627 objects degraded (5.263%) 2024-05-15T15:59:10.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:10 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:10.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:10 smithi053 ceph-mon[67220]: pgmap v138: 161 pgs: 12 active+undersized+degraded, 25 active+undersized, 124 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 33/627 objects degraded (5.263%) 2024-05-15T15:59:10.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:10 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:11.828 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 33/627 objects degraded (5.263%), 12 pgs degraded (PG_DEGRADED) 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:11.829 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 33/627 objects degraded (5.263%), 12 pgs degraded (PG_DEGRADED) 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:11.830 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:11.831 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:11 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 33/627 objects degraded (5.263%), 12 pgs degraded (PG_DEGRADED) 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:12.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:11 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:12.751 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/627 objects degraded (5.263%), 12 pgs degraded) 2024-05-15T15:59:12.751 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:12.751 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:12.751 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:12.751 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:12.751 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:12.751 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:12.753 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:59:12.613+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:12.753 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:59:12.753 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:12.753 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:12.753 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:12.754 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:12.755 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:12.755 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:12.755 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:12.755 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:12.755 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:12.755 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:12.755 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:12.755 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:12.756 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:12.757 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:12.758 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:12 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:13.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:59:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:59:13.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:59:13.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:59:13.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:59:13.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:59:13.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:59:13.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:59:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/627 objects degraded (5.263%), 12 pgs degraded) 2024-05-15T15:59:13.088 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:59:13.089 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:59:13.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:59:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:59:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:59:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:59:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:59:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:59:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:59:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:59:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:59:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.091 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:59:13.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:59:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:13.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:13.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:59:13.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:59:13.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:13.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:12 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/627 objects degraded (5.263%), 12 pgs degraded) 2024-05-15T15:59:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.132 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:59:13.133 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T15:59:13.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:13.136 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:12 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:14.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:14.086 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:14.086 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:14.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:14.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:14.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:14.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:14.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:14.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:14.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:14.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:14.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:14.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:14.090 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:14.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:14.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:14.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:14.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:59:14.094 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: Upgrade: Finalizing container_image settings 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: Upgrade: Complete! 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 924 B/s rd, 0 op/s 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:14.095 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: Upgrade: Finalizing container_image settings 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: Upgrade: Complete! 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 924 B/s rd, 0 op/s 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:14.096 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:13 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:14.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:14.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:14.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:14.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:14.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all nvmeof 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: Upgrade: Finalizing container_image settings 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: Upgrade: Complete! 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 924 B/s rd, 0 op/s 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:14.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:13 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:14 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:14 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:14 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:14 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:14.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:14 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:14.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:14 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:14 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:14 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:14 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:14 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:59:14.682+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:59:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:14 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:15.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:14 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:15.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:14 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:15.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:14 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:15.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:14 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[82531]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 0 op/s 2024-05-15T15:59:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:59:16.088 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:16.089 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[88099]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 0 op/s 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:16.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:16.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:16.092 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:16.093 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:16.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:16.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:16.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:16.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:16.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:16.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:16.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:16.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:16.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:16.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[82531]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[82531]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:16.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.096 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[88099]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-05-15T15:59:16.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[88099]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:59:16.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:16.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:16.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:16.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:16.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:15 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:16.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:15 smithi053 ceph-mon[67220]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 0 op/s 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:15 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:59:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:16.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:16.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:16.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:15 smithi053 ceph-mon[67220]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:15 smithi053 ceph-mon[67220]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:15 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:16.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:15 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:16.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:59:16] "GET /metrics HTTP/1.1" 200 38166 "" "Prometheus/2.43.0" 2024-05-15T15:59:17.032 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:16 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:17.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:16 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:17.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:16 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:59:16.698+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:17.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:17.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:17.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:17.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:17.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:17.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:17.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:59:17.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:17.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:59:17.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:17.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:17.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:17.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:17.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:17.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:17.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:17.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:17.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:17 smithi043 ceph-mon[88099]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-05-15T15:59:18.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:17 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:18.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:18.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:18.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:17 smithi043 ceph-mon[82531]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:17 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:18.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:18.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:18.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:18.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:18.095 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:18.095 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.095 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:17 smithi053 ceph-mon[67220]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-05-15T15:59:18.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:17 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:18.130 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:18.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:18.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:18.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:20.024 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:19 smithi043 ceph-mon[82531]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-05-15T15:59:20.025 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:19 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:20.025 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:19 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:20.025 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:19 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:20.025 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:19 smithi043 ceph-mon[88099]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-05-15T15:59:20.025 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:19 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:20.025 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:19 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:20.025 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:19 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:19 smithi053 ceph-mon[67220]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-05-15T15:59:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:19 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:19 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:19 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:20.743 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T15:59:21.063 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:20 smithi043 ceph-mon[82531]: from='client.44394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:21.063 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:20 smithi043 ceph-mon[88099]: from='client.44394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:21.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:20 smithi053 ceph-mon[67220]: from='client.44394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:22.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:21 smithi043 ceph-mon[82531]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:22.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:21 smithi043 ceph-mon[88099]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:22.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:21 smithi053 ceph-mon[67220]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:22.830 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:59:22.830 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (6m) 16s ago 28m 26.6M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:59:22.830 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (5m) 3m ago 28m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:59:22.830 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 16s ago 27m 65.0M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:59:22.830 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (7m) 3m ago 33m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:59:22.830 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (17m) 16s ago 34m 510M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:59:22.830 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (3m) 16s ago 34m 49.3M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (4m) 3m ago 33m 38.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (3m) 16s ago 33m 41.4M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (7m) 16s ago 29m 15.3M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (7m) 3m ago 29m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (109s) 16s ago 32m 44.0M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (87s) 16s ago 32m 43.5M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (22s) 16s ago 31m 12.1M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (31m) 16s ago 31m 56.9M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (31m) 3m ago 31m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (30m) 3m ago 30m 51.9M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (30m) 3m ago 30m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (30m) 3m ago 30m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:59:22.831 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (6m) 3m ago 29m 68.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:59:22.832 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (27m) 16s ago 27m 78.3M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:59:22.832 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (27m) 3m ago 27m 76.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:59:23.516 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-05-15T15:59:24.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:23 smithi043 ceph-mon[82531]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:24.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:23 smithi043 ceph-mon[82531]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:23 smithi043 ceph-mon[88099]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:23 smithi043 ceph-mon[88099]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:24.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:23 smithi053 ceph-mon[67220]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:24.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:23 smithi053 ceph-mon[67220]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:25.741 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:59:26.070 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:25 smithi043 ceph-mon[82531]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:59:26.070 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:25 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3374153607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:26.070 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:25 smithi043 ceph-mon[88099]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:59:26.070 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:25 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3374153607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:26.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:25 smithi053 ceph-mon[67220]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:59:26.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:25 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3374153607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:26.470 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-05-15T15:59:26.782 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:59:26] "GET /metrics HTTP/1.1" 200 38177 "" "Prometheus/2.43.0" 2024-05-15T15:59:27.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:26 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:27.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:26 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:27.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:26 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:27.948 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:27 smithi043 ceph-mon[82531]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:27.948 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:27 smithi043 ceph-mon[88099]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:28.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:27 smithi053 ceph-mon[67220]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:30.057 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:29 smithi043 ceph-mon[82531]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:30.057 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:29 smithi043 ceph-mon[82531]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:30.057 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:29 smithi043 ceph-mon[88099]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:30.057 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:29 smithi043 ceph-mon[88099]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:29 smithi053 ceph-mon[67220]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:29 smithi053 ceph-mon[67220]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:30.947 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:59:31.554 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade status' 2024-05-15T15:59:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:31 smithi043 ceph-mon[82531]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:32.093 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:31 smithi043 ceph-mon[88099]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:31 smithi053 ceph-mon[67220]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:33.547 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:59:33.548 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": null, 2024-05-15T15:59:33.548 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": false, 2024-05-15T15:59:33.548 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "", 2024-05-15T15:59:33.548 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:59:33.548 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": null, 2024-05-15T15:59:33.548 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "", 2024-05-15T15:59:33.548 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:59:33.548 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:59:34.092 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:33 smithi043 ceph-mon[82531]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:34.092 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:33 smithi043 ceph-mon[88099]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:34.116 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T15:59:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:33 smithi053 ceph-mon[67220]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:35.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:34 smithi043 ceph-mon[82531]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:35.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:34 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:35.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:34 smithi043 ceph-mon[88099]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:35.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:34 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:35.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:34 smithi053 ceph-mon[67220]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:35.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:34 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:36.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:35 smithi043 ceph-mon[82531]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:36.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:35 smithi043 ceph-mon[88099]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:36.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:35 smithi053 ceph-mon[67220]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:36.220 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T15:59:36.220 INFO:teuthology.orchestra.run.smithi043.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T15:59:36.220 INFO:teuthology.orchestra.run.smithi043.stdout: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T15:59:36.580 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:59:36] "GET /metrics HTTP/1.1" 200 38172 "" "Prometheus/2.43.0" 2024-05-15T15:59:36.891 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-05-15T15:59:37.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:36 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:37.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:36 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/528115046' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:59:37.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:36 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:37.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:36 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/528115046' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:59:37.211 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:36 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:37.211 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:36 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/528115046' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T15:59:38.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:37 smithi043 ceph-mon[82531]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:38.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:37 smithi043 ceph-mon[88099]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:37 smithi053 ceph-mon[67220]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:40.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:39 smithi053 ceph-mon[67220]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:40.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:39 smithi053 ceph-mon[67220]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:40.183 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:39 smithi043 ceph-mon[82531]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:40.183 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:39 smithi043 ceph-mon[82531]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:40.184 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:39 smithi043 ceph-mon[88099]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T15:59:40.184 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:39 smithi043 ceph-mon[88099]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:41.094 INFO:teuthology.orchestra.run.smithi043.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:59:41.754 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-15T15:59:42.316 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:42.316 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:59:42.316 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:42.316 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:42.316 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:42.317 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:42.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:42.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:42.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-05-15T15:59:42.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:42.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:42.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:42.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:42.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:42.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:42.318 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:42 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:42.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T15:59:42.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:42.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:42 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T15:59:43.256 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T15:59:43.081+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:43.256 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:59:43.256 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:43.257 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:43.258 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:43.258 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:43.258 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:43.258 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:43.258 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:43.258 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:43.258 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:43.258 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:43.259 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:43.260 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:43.261 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.378 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:44 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:44.378 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:44.378 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:44.378 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:44.378 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:44.378 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.378 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.378 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:44.378 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:44.379 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:44.380 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:44.381 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:44.382 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:44.382 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:44.382 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:44.382 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.382 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.382 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:44 smithi043 ceph-mon[82531]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:59:44.382 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:44 smithi043 ceph-mon[82531]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:44.382 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:44 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:44.382 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:44.382 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:44.383 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:44.384 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T15:59:44.385 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:44 smithi043 ceph-mon[88099]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:59:44.386 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:44 smithi043 ceph-mon[88099]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:44 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:59:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T15:59:44.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T15:59:44.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T15:59:44.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T15:59:44.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T15:59:44.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T15:59:44.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T15:59:44.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T15:59:44.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:44 smithi053 ceph-mon[67220]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T15:59:44.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:44 smithi053 ceph-mon[67220]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:44.869 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T15:59:45.584 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (6m) 39s ago 28m 26.6M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (6m) 4m ago 28m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 39s ago 28m 65.0M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (8m) 4m ago 33m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (18m) 39s ago 35m 510M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (4m) 39s ago 35m 49.3M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (5m) 4m ago 33m 38.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (3m) 39s ago 33m 41.4M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (7m) 39s ago 29m 15.3M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (7m) 4m ago 29m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (2m) 39s ago 33m 44.0M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (110s) 39s ago 32m 43.5M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (44s) 39s ago 32m 12.1M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (31m) 39s ago 31m 56.9M 2993M 17.2.0 e1d6a67b021e a6d6b5a56569 2024-05-15T15:59:45.585 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (31m) 4m ago 31m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T15:59:45.586 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (31m) 4m ago 31m 51.9M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T15:59:45.586 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (30m) 4m ago 30m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T15:59:45.586 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (30m) 4m ago 30m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T15:59:45.586 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (7m) 4m ago 29m 68.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T15:59:45.586 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (28m) 39s ago 28m 78.3M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T15:59:45.586 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (28m) 4m ago 28m 76.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T15:59:46.005 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:59:46.005 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T15:59:46.005 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T15:59:46.005 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:59:46.005 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T15:59:46.005 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T15:59:46.005 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 8 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T15:59:46.006 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:59:46.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='client.44436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:46.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-05-15T15:59:46.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:46.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1408503814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='client.44436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-05-15T15:59:46.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:46.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1408503814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:46.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:46 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-15T15:59:46.385 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T15:59:46.385 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T15:59:46.385 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T15:59:46.385 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-15T15:59:46.385 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T15:59:46.385 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "3/8 daemons upgraded", 2024-05-15T15:59:46.385 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading osd daemons", 2024-05-15T15:59:46.385 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T15:59:46.385 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='client.44436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='client.44442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1408503814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T15:59:46.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:46.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:46 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-15T15:59:46.759 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:59:46] "GET /metrics HTTP/1.1" 200 38172 "" "Prometheus/2.43.0" 2024-05-15T15:59:47.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[82531]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:47.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:47.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[82531]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:59:47.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[82531]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:59:47.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mgr 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mon 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all crash 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[82531]: Upgrade: osd.3 is safe to restart 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[88099]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[88099]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[88099]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mgr 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mon 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all crash 2024-05-15T15:59:47.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-15T15:59:47.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:47 smithi043 ceph-mon[88099]: Upgrade: osd.3 is safe to restart 2024-05-15T15:59:47.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:47 smithi053 ceph-mon[67220]: from='client.54385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:47.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:47 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:47.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:47 smithi053 ceph-mon[67220]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T15:59:47.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:47 smithi053 ceph-mon[67220]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T15:59:47.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:47 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mgr 2024-05-15T15:59:47.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:47 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mon 2024-05-15T15:59:47.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:47 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all crash 2024-05-15T15:59:47.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:47 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-15T15:59:47.532 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:47 smithi053 ceph-mon[67220]: Upgrade: osd.3 is safe to restart 2024-05-15T15:59:48.466 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:48 smithi043 ceph-mon[88099]: from='client.54397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:48.466 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:48 smithi043 ceph-mon[88099]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:48.466 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:48 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:48.466 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:48 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-15T15:59:48.466 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:48 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:48.466 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:48 smithi043 ceph-mon[82531]: from='client.54397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:48.467 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:48 smithi043 ceph-mon[82531]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:48.467 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:48 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:48.467 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:48 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-15T15:59:48.467 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:48 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:48.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:48 smithi053 ceph-mon[67220]: from='client.54397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T15:59:48.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:48 smithi053 ceph-mon[67220]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T15:59:48.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:48 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:48.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:48 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-15T15:59:48.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:48 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T15:59:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:49 smithi043 ceph-mon[82531]: Upgrade: Updating osd.3 2024-05-15T15:59:49.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:49 smithi043 ceph-mon[82531]: Deploying daemon osd.3 on smithi043 2024-05-15T15:59:49.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:49 smithi043 ceph-mon[88099]: Upgrade: Updating osd.3 2024-05-15T15:59:49.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:49 smithi043 ceph-mon[88099]: Deploying daemon osd.3 on smithi043 2024-05-15T15:59:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:49 smithi053 ceph-mon[67220]: Upgrade: Updating osd.3 2024-05-15T15:59:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:49 smithi053 ceph-mon[67220]: Deploying daemon osd.3 on smithi043 2024-05-15T15:59:50.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:50 smithi043 ceph-mon[82531]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-05-15T15:59:50.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:50 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:50.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:50 smithi043 ceph-mon[88099]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-05-15T15:59:50.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:50 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:50 smithi053 ceph-mon[67220]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 607 B/s rd, 0 op/s 2024-05-15T15:59:50.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:50 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T15:59:51.669 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:51 smithi043 systemd[1]: Stopping Ceph osd.3 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:59:52.086 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[40799]: 2024-05-15T15:59:51.666+0000 7f0cb65c3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T15:59:52.086 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[40799]: 2024-05-15T15:59:51.666+0000 7f0cb65c3700 -1 osd.3 122 *** Got signal Terminated *** 2024-05-15T15:59:52.086 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:51 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[40799]: 2024-05-15T15:59:51.666+0000 7f0cb65c3700 -1 osd.3 122 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T15:59:52.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:52 smithi043 ceph-mon[82531]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:59:52.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:52 smithi043 ceph-mon[82531]: osd.3 marked itself down and dead 2024-05-15T15:59:52.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:52 smithi043 ceph-mon[88099]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:59:52.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:52 smithi043 ceph-mon[88099]: osd.3 marked itself down and dead 2024-05-15T15:59:52.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:52 smithi053 ceph-mon[67220]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T15:59:52.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:52 smithi053 ceph-mon[67220]: osd.3 marked itself down and dead 2024-05-15T15:59:53.044 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:52 smithi043 podman[114293]: 2024-05-15 15:59:52.763157174 +0000 UTC m=+1.223294752 container died a6d6b5a56569085183edc2c9925ad0b6fa676846262e363866a4029998c527dc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, release=754, version=8, name=centos-stream, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, distribution-scope=public, RELEASE=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-15T15:59:53.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:53 smithi043 ceph-mon[82531]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:59:53.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:53 smithi043 ceph-mon[82531]: osdmap e123: 8 total, 7 up, 8 in 2024-05-15T15:59:53.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:53 smithi043 ceph-mon[88099]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:59:53.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:53 smithi043 ceph-mon[88099]: osdmap e123: 8 total, 7 up, 8 in 2024-05-15T15:59:53.337 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:53 smithi043 podman[114293]: 2024-05-15 15:59:53.157990587 +0000 UTC m=+1.618128159 container cleanup a6d6b5a56569085183edc2c9925ad0b6fa676846262e363866a4029998c527dc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, version=8) 2024-05-15T15:59:53.337 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:53 smithi043 bash[114293]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3 2024-05-15T15:59:53.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:53 smithi053 ceph-mon[67220]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T15:59:53.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:53 smithi053 ceph-mon[67220]: osdmap e123: 8 total, 7 up, 8 in 2024-05-15T15:59:53.814 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:53 smithi043 podman[114308]: 2024-05-15 15:59:53.434692147 +0000 UTC m=+0.672763866 container remove a6d6b5a56569085183edc2c9925ad0b6fa676846262e363866a4029998c527dc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, ceph=True, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, version=8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, distribution-scope=public, vcs-type=git, vendor=Red Hat, Inc.) 2024-05-15T15:59:54.086 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:53 smithi043 podman[114412]: 2024-05-15 15:59:53.714731501 +0000 UTC m=+0.018311980 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:59:54.086 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:53 smithi043 podman[114412]: 2024-05-15 15:59:53.835571178 +0000 UTC m=+0.139151641 container create b740312a9263f814a32055d46be879ee8eb646ca893828127f0caf7b8677394a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-05-15T15:59:54.086 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:54 smithi043 podman[114412]: 2024-05-15 15:59:54.038445077 +0000 UTC m=+0.342025542 container init b740312a9263f814a32055d46be879ee8eb646ca893828127f0caf7b8677394a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-05-15T15:59:54.087 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:54 smithi043 podman[114412]: 2024-05-15 15:59:54.042154047 +0000 UTC m=+0.345734524 container start b740312a9263f814a32055d46be879ee8eb646ca893828127f0caf7b8677394a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d) 2024-05-15T15:59:54.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:54 smithi043 ceph-mon[82531]: pgmap v167: 161 pgs: 14 peering, 21 stale+active+clean, 126 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-05-15T15:59:54.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:54 smithi043 ceph-mon[82531]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-05-15T15:59:54.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:54 smithi043 ceph-mon[82531]: osdmap e124: 8 total, 7 up, 8 in 2024-05-15T15:59:54.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:54 smithi043 ceph-mon[88099]: pgmap v167: 161 pgs: 14 peering, 21 stale+active+clean, 126 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-05-15T15:59:54.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:54 smithi043 ceph-mon[88099]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-05-15T15:59:54.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:54 smithi043 ceph-mon[88099]: osdmap e124: 8 total, 7 up, 8 in 2024-05-15T15:59:54.587 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:54 smithi043 podman[114412]: 2024-05-15 15:59:54.130044718 +0000 UTC m=+0.433625193 container attach b740312a9263f814a32055d46be879ee8eb646ca893828127f0caf7b8677394a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d) 2024-05-15T15:59:54.587 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:54 smithi043 podman[114412]: 2024-05-15 15:59:54.360483155 +0000 UTC m=+0.664063627 container died b740312a9263f814a32055d46be879ee8eb646ca893828127f0caf7b8677394a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d) 2024-05-15T15:59:54.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:54 smithi053 ceph-mon[67220]: pgmap v167: 161 pgs: 14 peering, 21 stale+active+clean, 126 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-05-15T15:59:54.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:54 smithi053 ceph-mon[67220]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-05-15T15:59:54.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:54 smithi053 ceph-mon[67220]: osdmap e124: 8 total, 7 up, 8 in 2024-05-15T15:59:55.253 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:54 smithi043 podman[114437]: 2024-05-15 15:59:54.93396995 +0000 UTC m=+0.567382121 container remove b740312a9263f814a32055d46be879ee8eb646ca893828127f0caf7b8677394a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS) 2024-05-15T15:59:55.253 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:54 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.3.service: Deactivated successfully. 2024-05-15T15:59:55.253 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:54 smithi043 systemd[1]: Stopped Ceph osd.3 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:59:55.253 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:54 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.3.service: Consumed 18.578s CPU time. 2024-05-15T15:59:55.253 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:55 smithi043 systemd[1]: Starting Ceph osd.3 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T15:59:55.503 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:55 smithi043 ceph-mon[82531]: Health check failed: Degraded data redundancy: 18/627 objects degraded (2.871%), 6 pgs degraded (PG_DEGRADED) 2024-05-15T15:59:55.503 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:55 smithi043 ceph-mon[88099]: Health check failed: Degraded data redundancy: 18/627 objects degraded (2.871%), 6 pgs degraded (PG_DEGRADED) 2024-05-15T15:59:55.504 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:55 smithi043 podman[114535]: 2024-05-15 15:59:55.29665652 +0000 UTC m=+0.018659835 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:59:55.504 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:55 smithi043 podman[114535]: 2024-05-15 15:59:55.401568583 +0000 UTC m=+0.123571891 container create 663dc15722fe824a83ba2d6ab993efb77efb6c6594c42299f5651f05e13eb133 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux ) 2024-05-15T15:59:55.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:55 smithi053 ceph-mon[67220]: Health check failed: Degraded data redundancy: 18/627 objects degraded (2.871%), 6 pgs degraded (PG_DEGRADED) 2024-05-15T15:59:55.819 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:55 smithi043 podman[114535]: 2024-05-15 15:59:55.629483023 +0000 UTC m=+0.351486331 container init 663dc15722fe824a83ba2d6ab993efb77efb6c6594c42299f5651f05e13eb133 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T15:59:55.819 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:55 smithi043 podman[114535]: 2024-05-15 15:59:55.633027396 +0000 UTC m=+0.355030716 container start 663dc15722fe824a83ba2d6ab993efb77efb6c6594c42299f5651f05e13eb133 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-65480-squid-de5210d) 2024-05-15T15:59:55.819 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:55 smithi043 podman[114535]: 2024-05-15 15:59:55.712708232 +0000 UTC m=+0.434711541 container attach 663dc15722fe824a83ba2d6ab993efb77efb6c6594c42299f5651f05e13eb133 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d) 2024-05-15T15:59:56.510 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:56 smithi043 ceph-mon[82531]: pgmap v169: 161 pgs: 6 active+undersized+degraded, 14 peering, 15 stale+active+clean, 8 active+undersized, 118 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 18/627 objects degraded (2.871%) 2024-05-15T15:59:56.511 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:56 smithi043 ceph-mon[88099]: pgmap v169: 161 pgs: 6 active+undersized+degraded, 14 peering, 15 stale+active+clean, 8 active+undersized, 118 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 18/627 objects degraded (2.871%) 2024-05-15T15:59:56.511 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 15:59:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:15:59:56] "GET /metrics HTTP/1.1" 200 38171 "" "Prometheus/2.43.0" 2024-05-15T15:59:56.511 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate[114549]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:59:56.511 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate[114549]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:59:56.511 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 bash[114535]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:59:56.511 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 bash[114535]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T15:59:56.511 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 bash[114535]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 bash[114535]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 bash[114535]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 bash[114535]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 bash[114535]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 bash[114535]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 bash[114535]: --> ceph-volume raw activate successful for osd ID: 3 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate[114549]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate[114549]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate[114549]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate[114549]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate[114549]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate[114549]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate[114549]: --> ceph-volume raw activate successful for osd ID: 3 2024-05-15T15:59:56.512 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 podman[114535]: 2024-05-15 15:59:56.159240153 +0000 UTC m=+0.881243468 container died 663dc15722fe824a83ba2d6ab993efb77efb6c6594c42299f5651f05e13eb133 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-15T15:59:56.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:56 smithi053 ceph-mon[67220]: pgmap v169: 161 pgs: 6 active+undersized+degraded, 14 peering, 15 stale+active+clean, 8 active+undersized, 118 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 18/627 objects degraded (2.871%) 2024-05-15T15:59:57.183 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:56 smithi043 podman[114673]: 2024-05-15 15:59:56.858348081 +0000 UTC m=+0.692902878 container remove 663dc15722fe824a83ba2d6ab993efb77efb6c6594c42299f5651f05e13eb133 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0) 2024-05-15T15:59:57.183 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:57 smithi043 podman[114732]: 2024-05-15 15:59:57.075977877 +0000 UTC m=+0.107883416 container create 258344f90360e1e5e457c9699138d6ee8e5a6457b470714fb20d68b3e536d6d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=) 2024-05-15T15:59:57.184 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:57 smithi043 podman[114732]: 2024-05-15 15:59:56.986350478 +0000 UTC m=+0.018255987 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T15:59:57.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:57 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:57.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:57 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:57.586 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:57 smithi043 podman[114732]: 2024-05-15 15:59:57.303680577 +0000 UTC m=+0.335586078 container init 258344f90360e1e5e457c9699138d6ee8e5a6457b470714fb20d68b3e536d6d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, ceph=True) 2024-05-15T15:59:57.586 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:57 smithi043 podman[114732]: 2024-05-15 15:59:57.307122686 +0000 UTC m=+0.339028184 container start 258344f90360e1e5e457c9699138d6ee8e5a6457b470714fb20d68b3e536d6d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-05-15T15:59:57.586 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:57 smithi043 ceph-osd[114752]: -- 172.21.15.43:0/3276245031 <== mon.1 v2:172.21.15.43:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560265481680 con 0x56026544b000 2024-05-15T15:59:57.587 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:57 smithi043 bash[114732]: 258344f90360e1e5e457c9699138d6ee8e5a6457b470714fb20d68b3e536d6d0 2024-05-15T15:59:57.587 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:57 smithi043 systemd[1]: Started Ceph osd.3 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T15:59:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:57 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T15:59:58.259 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 15:59:57 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[114748]: 2024-05-15T15:59:57.852+0000 7f770da437c0 -1 Falling back to public interface 2024-05-15T15:59:58.514 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:58 smithi043 ceph-mon[88099]: pgmap v170: 161 pgs: 22 active+undersized+degraded, 14 peering, 30 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 61/627 objects degraded (9.729%) 2024-05-15T15:59:58.515 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:58 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:58.515 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:58 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:58.515 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:58 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:58.515 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:58 smithi043 ceph-mon[82531]: pgmap v170: 161 pgs: 22 active+undersized+degraded, 14 peering, 30 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 61/627 objects degraded (9.729%) 2024-05-15T15:59:58.515 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:58 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:58.515 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:58 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:58.515 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:58 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:58 smithi053 ceph-mon[67220]: pgmap v170: 161 pgs: 22 active+undersized+degraded, 14 peering, 30 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 61/627 objects degraded (9.729%) 2024-05-15T15:59:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:58 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:58 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T15:59:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:58 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T15:59:59.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 15:59:59 smithi043 ceph-mon[82531]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-05-15T15:59:59.568 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 15:59:59 smithi043 ceph-mon[88099]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-05-15T15:59:59.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 15:59:59 smithi053 ceph-mon[67220]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-05-15T16:00:00.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pgmap v171: 161 pgs: 28 active+undersized+degraded, 38 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-15T16:00:00.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 84/627 objects degraded (13.397%), 28 pgs degraded 2024-05-15T16:00:00.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T16:00:00.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T16:00:00.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: [WRN] OSD_DOWN: 1 osds down 2024-05-15T16:00:00.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: osd.3 (root=default,host=smithi043) is down 2024-05-15T16:00:00.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: [WRN] PG_DEGRADED: Degraded data redundancy: 84/627 objects degraded (13.397%), 28 pgs degraded 2024-05-15T16:00:00.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-05-15T16:00:00.566 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 2.a is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 2.d is active+undersized+degraded, acting [1,4] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 2.e is active+undersized+degraded, acting [2,7] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 2.12 is active+undersized+degraded, acting [5,2] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.b is active+undersized+degraded, acting [0,4] 2024-05-15T16:00:00.567 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.c is active+undersized+degraded, acting [5,6] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 4.9 is active+undersized+degraded, acting [4,1] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 4.15 is active+undersized+degraded, acting [5,7] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 5.1c is active+undersized+degraded, acting [4,2] 2024-05-15T16:00:00.568 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[82531]: pg 6.c is active+undersized+degraded, acting [6,5] 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pgmap v171: 161 pgs: 28 active+undersized+degraded, 38 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 84/627 objects degraded (13.397%), 28 pgs degraded 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: [WRN] OSD_DOWN: 1 osds down 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: osd.3 (root=default,host=smithi043) is down 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: [WRN] PG_DEGRADED: Degraded data redundancy: 84/627 objects degraded (13.397%), 28 pgs degraded 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 2.a is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 2.d is active+undersized+degraded, acting [1,4] 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 2.e is active+undersized+degraded, acting [2,7] 2024-05-15T16:00:00.569 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 2.12 is active+undersized+degraded, acting [5,2] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.b is active+undersized+degraded, acting [0,4] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.c is active+undersized+degraded, acting [5,6] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-05-15T16:00:00.570 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-05-15T16:00:00.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-05-15T16:00:00.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-05-15T16:00:00.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-05-15T16:00:00.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 4.9 is active+undersized+degraded, acting [4,1] 2024-05-15T16:00:00.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 4.15 is active+undersized+degraded, acting [5,7] 2024-05-15T16:00:00.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 5.1c is active+undersized+degraded, acting [4,2] 2024-05-15T16:00:00.571 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:00 smithi043 ceph-mon[88099]: pg 6.c is active+undersized+degraded, acting [6,5] 2024-05-15T16:00:00.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pgmap v171: 161 pgs: 28 active+undersized+degraded, 38 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-15T16:00:00.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 84/627 objects degraded (13.397%), 28 pgs degraded 2024-05-15T16:00:00.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T16:00:00.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T16:00:00.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: [WRN] OSD_DOWN: 1 osds down 2024-05-15T16:00:00.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: osd.3 (root=default,host=smithi043) is down 2024-05-15T16:00:00.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: [WRN] PG_DEGRADED: Degraded data redundancy: 84/627 objects degraded (13.397%), 28 pgs degraded 2024-05-15T16:00:00.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-05-15T16:00:00.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 2.a is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 2.d is active+undersized+degraded, acting [1,4] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 2.e is active+undersized+degraded, acting [2,7] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 2.12 is active+undersized+degraded, acting [5,2] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.b is active+undersized+degraded, acting [0,4] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.c is active+undersized+degraded, acting [5,6] 2024-05-15T16:00:00.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 4.9 is active+undersized+degraded, acting [4,1] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 4.15 is active+undersized+degraded, acting [5,7] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 5.1c is active+undersized+degraded, acting [4,2] 2024-05-15T16:00:00.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:00 smithi053 ceph-mon[67220]: pg 6.c is active+undersized+degraded, acting [6,5] 2024-05-15T16:00:00.836 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:00:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[114748]: 2024-05-15T16:00:00.561+0000 7f770da437c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-05-15T16:00:00.836 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:00:00 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[114748]: 2024-05-15T16:00:00.591+0000 7f770da437c0 -1 osd.3 122 log_to_monitors true 2024-05-15T16:00:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:01 smithi043 ceph-mon[82531]: from='osd.3 [v2:172.21.15.43:6826/1990659532,v1:172.21.15.43:6827/1990659532]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-15T16:00:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:01 smithi043 ceph-mon[82531]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-15T16:00:01.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:01 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 84/627 objects degraded (13.397%), 28 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:01 smithi043 ceph-mon[88099]: from='osd.3 [v2:172.21.15.43:6826/1990659532,v1:172.21.15.43:6827/1990659532]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-15T16:00:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:01 smithi043 ceph-mon[88099]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-15T16:00:01.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:01 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 84/627 objects degraded (13.397%), 28 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:01 smithi053 ceph-mon[67220]: from='osd.3 [v2:172.21.15.43:6826/1990659532,v1:172.21.15.43:6827/1990659532]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-15T16:00:01.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:01 smithi053 ceph-mon[67220]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-15T16:00:01.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:01 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 84/627 objects degraded (13.397%), 28 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:02.086 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:00:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[114748]: 2024-05-15T16:00:01.594+0000 7f76fae92700 -1 osd.3 122 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T16:00:02.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[82531]: pgmap v172: 161 pgs: 28 active+undersized+degraded, 38 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-15T16:00:02.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[82531]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-15T16:00:02.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[82531]: osdmap e125: 8 total, 7 up, 8 in 2024-05-15T16:00:02.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[82531]: from='osd.3 [v2:172.21.15.43:6826/1990659532,v1:172.21.15.43:6827/1990659532]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[82531]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[82531]: from='osd.3 ' entity='osd.3' 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[88099]: pgmap v172: 161 pgs: 28 active+undersized+degraded, 38 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[88099]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[88099]: osdmap e125: 8 total, 7 up, 8 in 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[88099]: from='osd.3 [v2:172.21.15.43:6826/1990659532,v1:172.21.15.43:6827/1990659532]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[88099]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[88099]: from='osd.3 ' entity='osd.3' 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:02.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:02 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:02 smithi053 ceph-mon[67220]: pgmap v172: 161 pgs: 28 active+undersized+degraded, 38 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 84/627 objects degraded (13.397%) 2024-05-15T16:00:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:02 smithi053 ceph-mon[67220]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-15T16:00:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:02 smithi053 ceph-mon[67220]: osdmap e125: 8 total, 7 up, 8 in 2024-05-15T16:00:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:02 smithi053 ceph-mon[67220]: from='osd.3 [v2:172.21.15.43:6826/1990659532,v1:172.21.15.43:6827/1990659532]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T16:00:02.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:02 smithi053 ceph-mon[67220]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi043", "root=default"]}]: dispatch 2024-05-15T16:00:02.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:02 smithi053 ceph-mon[67220]: from='osd.3 ' entity='osd.3' 2024-05-15T16:00:02.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:02 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:02.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:02 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[82531]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:00:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[82531]: osd.3 [v2:172.21.15.43:6826/1990659532,v1:172.21.15.43:6827/1990659532] boot 2024-05-15T16:00:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[82531]: osdmap e126: 8 total, 8 up, 8 in 2024-05-15T16:00:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T16:00:03.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:03.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:03.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[88099]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:00:03.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[88099]: osd.3 [v2:172.21.15.43:6826/1990659532,v1:172.21.15.43:6827/1990659532] boot 2024-05-15T16:00:03.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[88099]: osdmap e126: 8 total, 8 up, 8 in 2024-05-15T16:00:03.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T16:00:03.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:03.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:03 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:03 smithi053 ceph-mon[67220]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:00:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:03 smithi053 ceph-mon[67220]: osd.3 [v2:172.21.15.43:6826/1990659532,v1:172.21.15.43:6827/1990659532] boot 2024-05-15T16:00:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:03 smithi053 ceph-mon[67220]: osdmap e126: 8 total, 8 up, 8 in 2024-05-15T16:00:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:03 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-15T16:00:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:03 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:03.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:03 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:04.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:04 smithi043 ceph-mon[82531]: pgmap v175: 161 pgs: 8 peering, 24 active+undersized+degraded, 34 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-05-15T16:00:04.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:04 smithi043 ceph-mon[82531]: osdmap e127: 8 total, 8 up, 8 in 2024-05-15T16:00:04.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:04 smithi043 ceph-mon[88099]: pgmap v175: 161 pgs: 8 peering, 24 active+undersized+degraded, 34 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-05-15T16:00:04.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:04 smithi043 ceph-mon[88099]: osdmap e127: 8 total, 8 up, 8 in 2024-05-15T16:00:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:04 smithi053 ceph-mon[67220]: pgmap v175: 161 pgs: 8 peering, 24 active+undersized+degraded, 34 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 69/627 objects degraded (11.005%) 2024-05-15T16:00:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:04 smithi053 ceph-mon[67220]: osdmap e127: 8 total, 8 up, 8 in 2024-05-15T16:00:05.996 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:05.996 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:05.996 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:05 smithi043 ceph-mon[82531]: pgmap v177: 161 pgs: 8 peering, 20 active+undersized+degraded, 30 active+undersized, 103 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 59/627 objects degraded (9.410%) 2024-05-15T16:00:05.996 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:05.997 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:05.997 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:05 smithi043 ceph-mon[88099]: pgmap v177: 161 pgs: 8 peering, 20 active+undersized+degraded, 30 active+undersized, 103 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 59/627 objects degraded (9.410%) 2024-05-15T16:00:06.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:06.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:06.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:05 smithi053 ceph-mon[67220]: pgmap v177: 161 pgs: 8 peering, 20 active+undersized+degraded, 30 active+undersized, 103 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 59/627 objects degraded (9.410%) 2024-05-15T16:00:06.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:00:06] "GET /metrics HTTP/1.1" 200 38042 "" "Prometheus/2.43.0" 2024-05-15T16:00:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:06 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:06 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 59/627 objects degraded (9.410%), 20 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:06 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:06 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 59/627 objects degraded (9.410%), 20 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:06 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:06 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 59/627 objects degraded (9.410%), 20 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:07.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:07.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: pgmap v178: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: pgmap v179: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:07.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:07.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:00:07.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: pgmap v178: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:07.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: pgmap v179: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:07.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:07.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:00:07.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:07.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:07.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:07.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:07 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:00:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:00:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: pgmap v178: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: pgmap v179: 161 pgs: 8 peering, 153 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:00:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:08.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:07 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:00:09.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:08 smithi043 ceph-mon[82531]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 59/627 objects degraded (9.410%), 20 pgs degraded) 2024-05-15T16:00:09.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:08 smithi043 ceph-mon[88099]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 59/627 objects degraded (9.410%), 20 pgs degraded) 2024-05-15T16:00:09.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:00:08.839+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:09.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:00:09.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:09.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:09.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:09.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:00:09.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:09.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:09.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:09.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:08 smithi053 ceph-mon[67220]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 59/627 objects degraded (9.410%), 20 pgs degraded) 2024-05-15T16:00:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[88099]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:10.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:00:10.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:10.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:10.090 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[82531]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:10.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:00:10.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[82531]: Upgrade: osd.4 is safe to restart 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:10.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:10.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:10.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:10.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-15T16:00:10.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-15T16:00:10.096 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:09 smithi043 ceph-mon[88099]: Upgrade: osd.4 is safe to restart 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:09 smithi053 ceph-mon[67220]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:09 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:10.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:00:10.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:10.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:10.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:10.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:09 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:10.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:09 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:10.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:09 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:10.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:09 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:10.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:09 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-15T16:00:10.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:09 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-15T16:00:10.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:09 smithi053 ceph-mon[67220]: Upgrade: osd.4 is safe to restart 2024-05-15T16:00:11.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:11 smithi053 ceph-mon[67220]: Upgrade: Updating osd.4 2024-05-15T16:00:11.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:11 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:11.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:11 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-15T16:00:11.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:11 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:11.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:11 smithi053 ceph-mon[67220]: Deploying daemon osd.4 on smithi053 2024-05-15T16:00:11.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:11 smithi053 ceph-mon[67220]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[82531]: Upgrade: Updating osd.4 2024-05-15T16:00:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-15T16:00:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[82531]: Deploying daemon osd.4 on smithi053 2024-05-15T16:00:12.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[82531]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:12.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[88099]: Upgrade: Updating osd.4 2024-05-15T16:00:12.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:12.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-15T16:00:12.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:12.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[88099]: Deploying daemon osd.4 on smithi053 2024-05-15T16:00:12.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:11 smithi043 ceph-mon[88099]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:13.988 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:13 smithi053 ceph-mon[67220]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:14.276 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:13 smithi053 systemd[1]: Stopping Ceph osd.4 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:00:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:13 smithi043 ceph-mon[82531]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:14.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:13 smithi043 ceph-mon[88099]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:14.630 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[28554]: 2024-05-15T16:00:14.273+0000 7f949cbef700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:00:14.631 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[28554]: 2024-05-15T16:00:14.273+0000 7f949cbef700 -1 osd.4 127 *** Got signal Terminated *** 2024-05-15T16:00:14.631 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[28554]: 2024-05-15T16:00:14.273+0000 7f949cbef700 -1 osd.4 127 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:00:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:14 smithi053 ceph-mon[67220]: osd.4 marked itself down and dead 2024-05-15T16:00:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:14 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:15.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:14 smithi043 ceph-mon[82531]: osd.4 marked itself down and dead 2024-05-15T16:00:15.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:14 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:14 smithi043 ceph-mon[88099]: osd.4 marked itself down and dead 2024-05-15T16:00:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:14 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:15.569 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:15 smithi053 podman[77267]: 2024-05-15 16:00:15.248143863 +0000 UTC m=+1.104002325 container died 9e41422ae93cd9e7fb0dd21ab9e61ddf15a627edad2ca7e6639fc2f2606794d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, version=8, vendor=Red Hat, Inc.) 2024-05-15T16:00:15.842 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:15 smithi053 podman[77267]: 2024-05-15 16:00:15.665434282 +0000 UTC m=+1.521292742 container cleanup 9e41422ae93cd9e7fb0dd21ab9e61ddf15a627edad2ca7e6639fc2f2606794d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4, GIT_CLEAN=True, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, distribution-scope=public, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, version=8, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, architecture=x86_64, ceph=True, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-15T16:00:15.843 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:15 smithi053 bash[77267]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4 2024-05-15T16:00:16.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:15 smithi053 ceph-mon[67220]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:15 smithi053 ceph-mon[67220]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:00:16.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:15 smithi053 ceph-mon[67220]: osdmap e128: 8 total, 7 up, 8 in 2024-05-15T16:00:16.131 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:15 smithi053 podman[77286]: 2024-05-15 16:00:15.883502604 +0000 UTC m=+0.629832146 container remove 9e41422ae93cd9e7fb0dd21ab9e61ddf15a627edad2ca7e6639fc2f2606794d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4, io.openshift.expose-services=, vcs-type=git, vendor=Red Hat, Inc., name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, architecture=x86_64, distribution-scope=public, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, release=754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux ) 2024-05-15T16:00:16.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:15 smithi043 ceph-mon[82531]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:16.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:15 smithi043 ceph-mon[82531]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:00:16.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:15 smithi043 ceph-mon[82531]: osdmap e128: 8 total, 7 up, 8 in 2024-05-15T16:00:16.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:15 smithi043 ceph-mon[88099]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:00:16.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:15 smithi043 ceph-mon[88099]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:00:16.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:15 smithi043 ceph-mon[88099]: osdmap e128: 8 total, 7 up, 8 in 2024-05-15T16:00:16.614 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:16 smithi053 podman[77375]: 2024-05-15 16:00:16.220913614 +0000 UTC m=+0.017891223 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:00:16.614 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:16 smithi053 podman[77375]: 2024-05-15 16:00:16.326758711 +0000 UTC m=+0.123736352 container create b5c24ca6a46fd478e465e5cab44bd33f50db50ddf27e494d3c5cb5b73f3eb7ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-05-15T16:00:16.614 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:16 smithi053 podman[77375]: 2024-05-15 16:00:16.529486518 +0000 UTC m=+0.326464122 container init b5c24ca6a46fd478e465e5cab44bd33f50db50ddf27e494d3c5cb5b73f3eb7ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-05-15T16:00:16.614 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:16 smithi053 podman[77375]: 2024-05-15 16:00:16.533346171 +0000 UTC m=+0.330323780 container start b5c24ca6a46fd478e465e5cab44bd33f50db50ddf27e494d3c5cb5b73f3eb7ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:00:16.614 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:16 smithi053 podman[77375]: 2024-05-15 16:00:16.612447079 +0000 UTC m=+0.409424693 container attach b5c24ca6a46fd478e465e5cab44bd33f50db50ddf27e494d3c5cb5b73f3eb7ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0) 2024-05-15T16:00:16.765 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T16:00:16.766 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:00:16] "GET /metrics HTTP/1.1" 200 38042 "" "Prometheus/2.43.0" 2024-05-15T16:00:16.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:16 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:16.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:16 smithi053 ceph-mon[67220]: osdmap e129: 8 total, 7 up, 8 in 2024-05-15T16:00:16.881 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:16 smithi053 podman[77375]: 2024-05-15 16:00:16.840014402 +0000 UTC m=+0.636992001 container died b5c24ca6a46fd478e465e5cab44bd33f50db50ddf27e494d3c5cb5b73f3eb7ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514) 2024-05-15T16:00:17.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:16 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:17.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:16 smithi043 ceph-mon[82531]: osdmap e129: 8 total, 7 up, 8 in 2024-05-15T16:00:17.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:16 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:17.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:16 smithi043 ceph-mon[88099]: osdmap e129: 8 total, 7 up, 8 in 2024-05-15T16:00:17.387 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:17 smithi053 podman[77401]: 2024-05-15 16:00:17.383002183 +0000 UTC m=+0.532922720 container remove b5c24ca6a46fd478e465e5cab44bd33f50db50ddf27e494d3c5cb5b73f3eb7ff (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:00:17.451 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T16:00:17.451 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (7m) 15s ago 29m 26.6M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T16:00:17.451 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (6m) 4m ago 29m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T16:00:17.451 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 15s ago 28m 65.1M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T16:00:17.451 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (8m) 4m ago 33m 449M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T16:00:17.451 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (18m) 15s ago 35m 512M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (4m) 15s ago 35m 52.0M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (5m) 4m ago 34m 38.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (4m) 15s ago 34m 42.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (8m) 15s ago 30m 15.1M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (8m) 4m ago 30m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (2m) 15s ago 33m 45.6M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (2m) 15s ago 33m 44.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (76s) 15s ago 32m 49.7M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (20s) 15s ago 32m 12.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 258344f90360 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (32m) 4m ago 32m 54.2M 3249M 17.2.0 e1d6a67b021e 9e41422ae93c 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (31m) 4m ago 31m 51.9M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (31m) 4m ago 31m 55.0M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (30m) 4m ago 30m 55.6M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (7m) 4m ago 30m 68.8M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (28m) 15s ago 28m 78.6M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T16:00:17.452 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (28m) 4m ago 28m 76.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T16:00:17.849 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:17 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.4.service: Deactivated successfully. 2024-05-15T16:00:17.849 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:17 smithi053 systemd[1]: Stopped Ceph osd.4 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:00:17.849 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:17 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.4.service: Consumed 18.413s CPU time. 2024-05-15T16:00:17.849 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:17 smithi053 systemd[1]: Starting Ceph osd.4 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:00:17.865 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:00:17.865 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T16:00:17.865 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T16:00:17.865 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:00:17.865 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T16:00:17.865 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:00:17.865 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 4 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 9 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T16:00:17.866 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:00:18.114 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:17 smithi053 ceph-mon[67220]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:18.114 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:17 smithi053 ceph-mon[67220]: pgmap v186: 161 pgs: 2 active+undersized+degraded, 38 peering, 5 stale+active+clean, 8 active+undersized, 108 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-05-15T16:00:18.114 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:17 smithi053 ceph-mon[67220]: Health check failed: Reduced data availability: 8 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-05-15T16:00:18.114 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:17 smithi053 ceph-mon[67220]: Health check failed: Degraded data redundancy: 5/627 objects degraded (0.797%), 2 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:18.115 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:17 smithi053 ceph-mon[67220]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:18.115 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:17 smithi053 podman[77503]: 2024-05-15 16:00:17.750479071 +0000 UTC m=+0.018218871 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:00:18.115 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:17 smithi053 podman[77503]: 2024-05-15 16:00:17.850593863 +0000 UTC m=+0.118333645 container create 8973d06d919880093f324379c9a41caeae0bc8759cc09a143d4ba53806e25ac7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, ceph=True, GIT_BRANCH=HEAD) 2024-05-15T16:00:18.115 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 podman[77503]: 2024-05-15 16:00:18.112038474 +0000 UTC m=+0.379778266 container init 8973d06d919880093f324379c9a41caeae0bc8759cc09a143d4ba53806e25ac7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True) 2024-05-15T16:00:18.211 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:00:18.211 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T16:00:18.211 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T16:00:18.211 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-15T16:00:18.211 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T16:00:18.211 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "4/8 daemons upgraded", 2024-05-15T16:00:18.211 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading osd daemons", 2024-05-15T16:00:18.211 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T16:00:18.211 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:00:18.212 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:17 smithi043 ceph-mon[82531]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:18.212 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:17 smithi043 ceph-mon[82531]: pgmap v186: 161 pgs: 2 active+undersized+degraded, 38 peering, 5 stale+active+clean, 8 active+undersized, 108 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-05-15T16:00:18.212 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:17 smithi043 ceph-mon[82531]: Health check failed: Reduced data availability: 8 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-05-15T16:00:18.212 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:17 smithi043 ceph-mon[82531]: Health check failed: Degraded data redundancy: 5/627 objects degraded (0.797%), 2 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:18.212 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:17 smithi043 ceph-mon[82531]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:18.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:17 smithi043 ceph-mon[88099]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:18.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:17 smithi043 ceph-mon[88099]: pgmap v186: 161 pgs: 2 active+undersized+degraded, 38 peering, 5 stale+active+clean, 8 active+undersized, 108 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-05-15T16:00:18.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:17 smithi043 ceph-mon[88099]: Health check failed: Reduced data availability: 8 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-05-15T16:00:18.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:17 smithi043 ceph-mon[88099]: Health check failed: Degraded data redundancy: 5/627 objects degraded (0.797%), 2 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:18.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:17 smithi043 ceph-mon[88099]: from='client.44478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:18.380 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 podman[77503]: 2024-05-15 16:00:18.116042633 +0000 UTC m=+0.383782414 container start 8973d06d919880093f324379c9a41caeae0bc8759cc09a143d4ba53806e25ac7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d) 2024-05-15T16:00:18.381 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 podman[77503]: 2024-05-15 16:00:18.195401899 +0000 UTC m=+0.463141699 container attach 8973d06d919880093f324379c9a41caeae0bc8759cc09a143d4ba53806e25ac7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, ceph=True, maintainer=Guillaume Abrioux ) 2024-05-15T16:00:18.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:18 smithi053 ceph-mon[67220]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:18.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:18 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/938943538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:18.881 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:18 smithi053 ceph-mon[67220]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:18.881 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate[77518]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:00:18.881 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate[77518]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:00:18.881 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 bash[77503]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:00:18.881 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 bash[77503]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 bash[77503]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 bash[77503]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 bash[77503]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 bash[77503]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 bash[77503]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 bash[77503]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 bash[77503]: --> ceph-volume raw activate successful for osd ID: 4 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate[77518]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate[77518]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate[77518]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate[77518]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate[77518]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-05-15T16:00:18.882 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate[77518]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-15T16:00:18.883 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate[77518]: --> ceph-volume raw activate successful for osd ID: 4 2024-05-15T16:00:18.883 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:18 smithi053 podman[77503]: 2024-05-15 16:00:18.64092156 +0000 UTC m=+0.908661352 container died 8973d06d919880093f324379c9a41caeae0bc8759cc09a143d4ba53806e25ac7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-05-15T16:00:19.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:18 smithi043 ceph-mon[88099]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:19.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:18 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/938943538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:19.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:18 smithi043 ceph-mon[88099]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:19.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:18 smithi043 ceph-mon[82531]: from='client.44484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:19.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:18 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/938943538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:19.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:18 smithi043 ceph-mon[82531]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:19.592 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:19 smithi053 podman[77650]: 2024-05-15 16:00:19.24524886 +0000 UTC m=+0.597768398 container remove 8973d06d919880093f324379c9a41caeae0bc8759cc09a143d4ba53806e25ac7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, org.label-schema.build-date=20240514) 2024-05-15T16:00:19.592 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:19 smithi053 podman[77709]: 2024-05-15 16:00:19.371830944 +0000 UTC m=+0.018146982 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:00:19.592 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:19 smithi053 podman[77709]: 2024-05-15 16:00:19.475579753 +0000 UTC m=+0.121895831 container create b342d0fab575f9eb661f0fc0c70891f44acec27de13b902837c1be1dbf8b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5) 2024-05-15T16:00:19.859 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:19 smithi053 podman[77709]: 2024-05-15 16:00:19.7034645 +0000 UTC m=+0.349780526 container init b342d0fab575f9eb661f0fc0c70891f44acec27de13b902837c1be1dbf8b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:00:19.859 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:19 smithi053 podman[77709]: 2024-05-15 16:00:19.707044451 +0000 UTC m=+0.353360485 container start b342d0fab575f9eb661f0fc0c70891f44acec27de13b902837c1be1dbf8b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, ceph=True) 2024-05-15T16:00:19.860 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:19 smithi053 ceph-osd[77728]: -- 172.21.15.53:0/3145041870 <== mon.2 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cbf7ee5680 con 0x55cbf7eaf000 2024-05-15T16:00:19.860 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:19 smithi053 bash[77709]: b342d0fab575f9eb661f0fc0c70891f44acec27de13b902837c1be1dbf8b1efc 2024-05-15T16:00:19.860 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:19 smithi053 systemd[1]: Started Ceph osd.4 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:00:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:19 smithi053 ceph-mon[67220]: pgmap v187: 161 pgs: 8 active+undersized+degraded, 38 peering, 3 stale+active+clean, 11 active+undersized, 101 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 30/627 objects degraded (4.785%) 2024-05-15T16:00:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:19 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:19 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:20.239 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:19 smithi043 ceph-mon[88099]: pgmap v187: 161 pgs: 8 active+undersized+degraded, 38 peering, 3 stale+active+clean, 11 active+undersized, 101 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 30/627 objects degraded (4.785%) 2024-05-15T16:00:20.239 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:19 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:20.239 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:19 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:20.239 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:19 smithi043 ceph-mon[82531]: pgmap v187: 161 pgs: 8 active+undersized+degraded, 38 peering, 3 stale+active+clean, 11 active+undersized, 101 active+clean; 585 KiB data, 173 MiB used, 715 GiB / 715 GiB avail; 30/627 objects degraded (4.785%) 2024-05-15T16:00:20.240 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:19 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:20.240 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:19 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:20.630 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:20 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:00:20.257+0000 7f88292b37c0 -1 Falling back to public interface 2024-05-15T16:00:21.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:20 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:20 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:21.261 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:20 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:21.262 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:20 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:21.262 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:21.262 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:20 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:21.262 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:20 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:21.262 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:22.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:21 smithi053 ceph-mon[67220]: pgmap v188: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 106/627 objects degraded (16.906%) 2024-05-15T16:00:22.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:21 smithi053 ceph-mon[67220]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 12 pgs peering) 2024-05-15T16:00:22.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:21 smithi043 ceph-mon[88099]: pgmap v188: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 106/627 objects degraded (16.906%) 2024-05-15T16:00:22.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:21 smithi043 ceph-mon[88099]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 12 pgs peering) 2024-05-15T16:00:22.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:21 smithi043 ceph-mon[82531]: pgmap v188: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 106/627 objects degraded (16.906%) 2024-05-15T16:00:22.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:21 smithi043 ceph-mon[82531]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 12 pgs peering) 2024-05-15T16:00:23.380 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:22 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:00:22.961+0000 7f88292b37c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-05-15T16:00:23.380 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:22 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:00:22.993+0000 7f88292b37c0 -1 osd.4 127 log_to_monitors true 2024-05-15T16:00:24.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:23 smithi053 ceph-mon[67220]: pgmap v189: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 106/627 objects degraded (16.906%) 2024-05-15T16:00:24.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:23 smithi053 ceph-mon[67220]: from='osd.4 [v2:172.21.15.53:6800/3024805138,v1:172.21.15.53:6801/3024805138]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T16:00:24.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:23 smithi053 ceph-mon[67220]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T16:00:24.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:23 smithi043 ceph-mon[82531]: pgmap v189: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 106/627 objects degraded (16.906%) 2024-05-15T16:00:24.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:23 smithi043 ceph-mon[82531]: from='osd.4 [v2:172.21.15.53:6800/3024805138,v1:172.21.15.53:6801/3024805138]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T16:00:24.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:23 smithi043 ceph-mon[82531]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T16:00:24.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:23 smithi043 ceph-mon[88099]: pgmap v189: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 106/627 objects degraded (16.906%) 2024-05-15T16:00:24.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:23 smithi043 ceph-mon[88099]: from='osd.4 [v2:172.21.15.53:6800/3024805138,v1:172.21.15.53:6801/3024805138]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T16:00:24.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:23 smithi043 ceph-mon[88099]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-15T16:00:24.630 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:00:24 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:00:24.180+0000 7f8816702700 -1 osd.4 127 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T16:00:25.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:24 smithi053 ceph-mon[67220]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-15T16:00:25.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:24 smithi053 ceph-mon[67220]: from='osd.4 [v2:172.21.15.53:6800/3024805138,v1:172.21.15.53:6801/3024805138]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:00:25.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:24 smithi053 ceph-mon[67220]: osdmap e130: 8 total, 7 up, 8 in 2024-05-15T16:00:25.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:24 smithi053 ceph-mon[67220]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:00:25.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:25.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:25.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:24 smithi053 ceph-mon[67220]: from='osd.4 ' entity='osd.4' 2024-05-15T16:00:25.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[82531]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-15T16:00:25.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[82531]: from='osd.4 [v2:172.21.15.53:6800/3024805138,v1:172.21.15.53:6801/3024805138]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:00:25.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[82531]: osdmap e130: 8 total, 7 up, 8 in 2024-05-15T16:00:25.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[82531]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:00:25.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:25.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:25.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[82531]: from='osd.4 ' entity='osd.4' 2024-05-15T16:00:25.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[88099]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-15T16:00:25.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[88099]: from='osd.4 [v2:172.21.15.53:6800/3024805138,v1:172.21.15.53:6801/3024805138]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:00:25.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[88099]: osdmap e130: 8 total, 7 up, 8 in 2024-05-15T16:00:25.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[88099]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:00:25.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:25.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:25.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:24 smithi043 ceph-mon[88099]: from='osd.4 ' entity='osd.4' 2024-05-15T16:00:26.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:25 smithi053 ceph-mon[67220]: pgmap v191: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 188 MiB used, 715 GiB / 715 GiB avail; 106/627 objects degraded (16.906%) 2024-05-15T16:00:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:25 smithi053 ceph-mon[67220]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:00:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:25 smithi053 ceph-mon[67220]: osd.4 [v2:172.21.15.53:6800/3024805138,v1:172.21.15.53:6801/3024805138] boot 2024-05-15T16:00:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:25 smithi053 ceph-mon[67220]: osdmap e131: 8 total, 8 up, 8 in 2024-05-15T16:00:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:25 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T16:00:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:25 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:25 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:25 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 106/627 objects degraded (16.906%), 27 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[82531]: pgmap v191: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 188 MiB used, 715 GiB / 715 GiB avail; 106/627 objects degraded (16.906%) 2024-05-15T16:00:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[82531]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:00:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[82531]: osd.4 [v2:172.21.15.53:6800/3024805138,v1:172.21.15.53:6801/3024805138] boot 2024-05-15T16:00:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[82531]: osdmap e131: 8 total, 8 up, 8 in 2024-05-15T16:00:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T16:00:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 106/627 objects degraded (16.906%), 27 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[88099]: pgmap v191: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 188 MiB used, 715 GiB / 715 GiB avail; 106/627 objects degraded (16.906%) 2024-05-15T16:00:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[88099]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:00:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[88099]: osd.4 [v2:172.21.15.53:6800/3024805138,v1:172.21.15.53:6801/3024805138] boot 2024-05-15T16:00:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[88099]: osdmap e131: 8 total, 8 up, 8 in 2024-05-15T16:00:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-15T16:00:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:25 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 106/627 objects degraded (16.906%), 27 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:26.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:00:26] "GET /metrics HTTP/1.1" 200 38250 "" "Prometheus/2.43.0" 2024-05-15T16:00:27.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:27 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:27.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:27 smithi053 ceph-mon[67220]: osdmap e132: 8 total, 8 up, 8 in 2024-05-15T16:00:27.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:27 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:27.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:27 smithi043 ceph-mon[82531]: osdmap e132: 8 total, 8 up, 8 in 2024-05-15T16:00:27.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:27 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:27.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:27 smithi043 ceph-mon[88099]: osdmap e132: 8 total, 8 up, 8 in 2024-05-15T16:00:28.290 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:28 smithi053 ceph-mon[67220]: pgmap v194: 161 pgs: 26 peering, 16 active+undersized+degraded, 22 active+undersized, 97 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 52/627 objects degraded (8.293%) 2024-05-15T16:00:28.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:28 smithi043 ceph-mon[82531]: pgmap v194: 161 pgs: 26 peering, 16 active+undersized+degraded, 22 active+undersized, 97 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 52/627 objects degraded (8.293%) 2024-05-15T16:00:28.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:28 smithi043 ceph-mon[88099]: pgmap v194: 161 pgs: 26 peering, 16 active+undersized+degraded, 22 active+undersized, 97 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 52/627 objects degraded (8.293%) 2024-05-15T16:00:29.781 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:29.782 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:29.782 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:29.782 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:00:29.782 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: pgmap v195: 161 pgs: 26 peering, 12 active+undersized+degraded, 19 active+undersized, 104 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s; 36/627 objects degraded (5.742%) 2024-05-15T16:00:29.782 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:29.782 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:00:29.782 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:29.782 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:29.782 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:29.782 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: pgmap v195: 161 pgs: 26 peering, 12 active+undersized+degraded, 19 active+undersized, 104 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s; 36/627 objects degraded (5.742%) 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:29.783 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:29 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:00:30.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:00:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: pgmap v195: 161 pgs: 26 peering, 12 active+undersized+degraded, 19 active+undersized, 104 active+clean; 585 KiB data, 189 MiB used, 715 GiB / 715 GiB avail; 873 B/s rd, 0 op/s; 36/627 objects degraded (5.742%) 2024-05-15T16:00:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:00:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:00:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:30.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:29 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:00:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:30 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:30 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:30 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:30 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:30.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:30 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:30.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:30 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:30.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:30 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:30.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:30 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:30.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:30 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:30.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:30 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:00:30.607+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:00:30.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:00:30.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:00:30.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:30.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:30.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:30.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:30.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:30.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:30.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:30.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:30 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:00:30.620+0000 7fbf8c6f9700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-05-15T16:00:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:30 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:30 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:30 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:30 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:31.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:30 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:31.939 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:31 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:31.939 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:31.939 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:31.939 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:31.939 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:31.939 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:31.939 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:31.940 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:00:31.941 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:31.942 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:31 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:31.943 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:00:31.944 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:31.945 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:31.946 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:31.947 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:31.947 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:31.947 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:31.947 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:31.947 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:31.947 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:31 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:31.947 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:31 smithi043 ceph-mon[88099]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-05-15T16:00:31.947 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:31 smithi043 ceph-mon[88099]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-05-15T16:00:31.947 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:31 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 36/627 objects degraded (5.742%), 12 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:31.947 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:31.948 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:31.948 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:31.948 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:31.948 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:31.948 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:31 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:31.948 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:31 smithi043 ceph-mon[82531]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-05-15T16:00:31.948 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:31 smithi043 ceph-mon[82531]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-05-15T16:00:31.948 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:31 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 36/627 objects degraded (5.742%), 12 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:31 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:00:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:00:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:31 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:32.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:31 smithi053 ceph-mon[67220]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-05-15T16:00:32.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:31 smithi053 ceph-mon[67220]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-05-15T16:00:32.135 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:31 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 36/627 objects degraded (5.742%), 12 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:32.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:32 smithi043 ceph-mon[82531]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/627 objects degraded (5.742%), 12 pgs degraded) 2024-05-15T16:00:32.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:32 smithi043 ceph-mon[88099]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/627 objects degraded (5.742%), 12 pgs degraded) 2024-05-15T16:00:33.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:32 smithi053 ceph-mon[67220]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/627 objects degraded (5.742%), 12 pgs degraded) 2024-05-15T16:00:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:33 smithi043 ceph-mon[82531]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T16:00:34.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:33 smithi043 ceph-mon[88099]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T16:00:34.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:33 smithi053 ceph-mon[67220]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T16:00:36.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:36.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:36.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:35 smithi043 ceph-mon[82531]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T16:00:36.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:36.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:36.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:35 smithi043 ceph-mon[88099]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T16:00:36.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:36.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:36.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:35 smithi053 ceph-mon[67220]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T16:00:36.648 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:00:36] "GET /metrics HTTP/1.1" 200 38260 "" "Prometheus/2.43.0" 2024-05-15T16:00:37.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:36 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:37.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:36 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:37.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:36 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:38.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:37 smithi043 ceph-mon[82531]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:00:38.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:37 smithi043 ceph-mon[88099]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:00:38.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:37 smithi053 ceph-mon[67220]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:00:40.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:39 smithi043 ceph-mon[82531]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-05-15T16:00:40.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:39 smithi043 ceph-mon[88099]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-05-15T16:00:40.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:39 smithi053 ceph-mon[67220]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-05-15T16:00:41.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:40 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:41.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:40 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:41.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:40 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:42.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:41 smithi043 ceph-mon[82531]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:42.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:41 smithi043 ceph-mon[88099]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:42.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:41 smithi053 ceph-mon[67220]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:44.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:43 smithi043 ceph-mon[82531]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:00:44.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:43 smithi043 ceph-mon[88099]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:00:44.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:43 smithi053 ceph-mon[67220]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:00:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:45 smithi043 ceph-mon[82531]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:46.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:46.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:45 smithi043 ceph-mon[88099]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:46.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:46.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:45 smithi053 ceph-mon[67220]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:46.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:46.791 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:00:46] "GET /metrics HTTP/1.1" 200 38260 "" "Prometheus/2.43.0" 2024-05-15T16:00:46.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:46 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:46.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:46 smithi053 ceph-mon[67220]: Upgrade: osd.5 is safe to restart 2024-05-15T16:00:46.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:46 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:47.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:46 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:47.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:46 smithi043 ceph-mon[88099]: Upgrade: osd.5 is safe to restart 2024-05-15T16:00:47.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:46 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:47.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:46 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-15T16:00:47.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:46 smithi043 ceph-mon[82531]: Upgrade: osd.5 is safe to restart 2024-05-15T16:00:47.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:46 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:48.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:47 smithi043 ceph-mon[88099]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:00:48.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:47 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:48.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:47 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-15T16:00:48.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:47 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:48.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:47 smithi043 ceph-mon[82531]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:00:48.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:47 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:48.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:47 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-15T16:00:48.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:47 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:48.108 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:47 smithi053 ceph-mon[67220]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:00:48.108 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:47 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:48.109 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:47 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-15T16:00:48.109 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:47 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:00:48.589 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T16:00:49.040 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:48 smithi053 ceph-mon[67220]: Upgrade: Updating osd.5 2024-05-15T16:00:49.040 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:48 smithi053 ceph-mon[67220]: Deploying daemon osd.5 on smithi053 2024-05-15T16:00:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:48 smithi043 ceph-mon[82531]: Upgrade: Updating osd.5 2024-05-15T16:00:49.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:48 smithi043 ceph-mon[82531]: Deploying daemon osd.5 on smithi053 2024-05-15T16:00:49.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:48 smithi043 ceph-mon[88099]: Upgrade: Updating osd.5 2024-05-15T16:00:49.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:48 smithi043 ceph-mon[88099]: Deploying daemon osd.5 on smithi053 2024-05-15T16:00:49.272 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T16:00:49.272 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (7m) 47s ago 29m 26.6M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (7m) 25s ago 29m 83.5M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 47s ago 29m 65.1M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (9m) 25s ago 34m 454M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (19m) 47s ago 36m 512M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (5m) 47s ago 36m 52.0M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (6m) 25s ago 34m 50.5M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (4m) 47s ago 34m 42.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (8m) 47s ago 30m 15.1M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (8m) 25s ago 30m 16.5M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (3m) 47s ago 34m 45.6M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (2m) 47s ago 33m 44.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (108s) 47s ago 33m 49.7M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (52s) 47s ago 32m 12.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 258344f90360 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (29s) 25s ago 32m 11.9M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 b342d0fab575 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (32m) 25s ago 32m 55.3M 3249M 17.2.0 e1d6a67b021e 94acc2d2ee77 2024-05-15T16:00:49.273 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (31m) 25s ago 31m 58.1M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T16:00:49.274 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (31m) 25s ago 31m 61.4M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T16:00:49.274 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (8m) 25s ago 30m 73.3M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T16:00:49.274 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (29m) 47s ago 29m 78.6M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T16:00:49.274 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (29m) 25s ago 29m 77.2M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T16:00:49.687 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:00:49.687 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T16:00:49.687 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T16:00:49.687 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:00:49.687 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T16:00:49.687 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:00:49.687 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 5 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 10 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T16:00:49.688 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:00:50.037 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:00:50.038 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T16:00:50.038 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T16:00:50.038 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-15T16:00:50.038 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T16:00:50.038 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "5/8 daemons upgraded", 2024-05-15T16:00:50.038 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading osd daemons", 2024-05-15T16:00:50.038 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T16:00:50.038 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:00:50.038 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[82531]: from='client.44508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:50.038 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[82531]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:50.038 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[82531]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:50.039 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[82531]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:50.039 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:50.039 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3879949444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:50.039 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[88099]: from='client.44508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:50.039 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[88099]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:50.039 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[88099]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:50.039 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[88099]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:50.039 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:50.039 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:49 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3879949444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:49 smithi053 ceph-mon[67220]: from='client.44508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:49 smithi053 ceph-mon[67220]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:49 smithi053 ceph-mon[67220]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:49 smithi053 ceph-mon[67220]: from='client.44520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:50.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:49 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:00:50.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:49 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3879949444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:00:51.046 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:50 smithi053 ceph-mon[67220]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:51.047 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:50 smithi053 systemd[1]: Stopping Ceph osd.5 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:00:51.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:50 smithi043 ceph-mon[82531]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:51.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:50 smithi043 ceph-mon[88099]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:00:51.380 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:51 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[31415]: 2024-05-15T16:00:51.043+0000 7f31313c7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:00:51.380 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:51 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[31415]: 2024-05-15T16:00:51.043+0000 7f31313c7700 -1 osd.5 132 *** Got signal Terminated *** 2024-05-15T16:00:51.380 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:51 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[31415]: 2024-05-15T16:00:51.043+0000 7f31313c7700 -1 osd.5 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:00:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:51 smithi043 ceph-mon[82531]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:52.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:51 smithi043 ceph-mon[82531]: osd.5 marked itself down and dead 2024-05-15T16:00:52.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:51 smithi043 ceph-mon[88099]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:52.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:51 smithi043 ceph-mon[88099]: osd.5 marked itself down and dead 2024-05-15T16:00:52.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:51 smithi053 ceph-mon[67220]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:00:52.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:51 smithi053 ceph-mon[67220]: osd.5 marked itself down and dead 2024-05-15T16:00:52.563 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:52 smithi053 podman[81768]: 2024-05-15 16:00:52.3018405 +0000 UTC m=+1.365878464 container died 94acc2d2ee779de050f05d247c200d5a92a0374b50229993c7b6a8d9f06d1720 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5, distribution-scope=public, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_CLEAN=True, architecture=x86_64, RELEASE=HEAD, ceph=True, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-15T16:00:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:52 smithi053 ceph-mon[67220]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:00:52.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:52 smithi053 ceph-mon[67220]: osdmap e133: 8 total, 7 up, 8 in 2024-05-15T16:00:52.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:52 smithi053 podman[81768]: 2024-05-15 16:00:52.661017863 +0000 UTC m=+1.725055815 container cleanup 94acc2d2ee779de050f05d247c200d5a92a0374b50229993c7b6a8d9f06d1720 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5, io.k8s.display-name=CentOS Stream 8, vcs-type=git, RELEASE=HEAD, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , version=8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, ceph=True, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754) 2024-05-15T16:00:52.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:52 smithi053 bash[81768]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5 2024-05-15T16:00:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:52 smithi043 ceph-mon[82531]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:00:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:52 smithi043 ceph-mon[82531]: osdmap e133: 8 total, 7 up, 8 in 2024-05-15T16:00:53.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:52 smithi043 ceph-mon[88099]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:00:53.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:52 smithi043 ceph-mon[88099]: osdmap e133: 8 total, 7 up, 8 in 2024-05-15T16:00:53.325 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:52 smithi053 podman[81787]: 2024-05-15 16:00:52.995923346 +0000 UTC m=+0.688128138 container remove 94acc2d2ee779de050f05d247c200d5a92a0374b50229993c7b6a8d9f06d1720 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, ceph=True, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, version=8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, architecture=x86_64, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, RELEASE=HEAD) 2024-05-15T16:00:53.630 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:53 smithi053 podman[81874]: 2024-05-15 16:00:53.22652593 +0000 UTC m=+0.018454958 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:00:53.631 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:53 smithi053 podman[81874]: 2024-05-15 16:00:53.330604818 +0000 UTC m=+0.122533864 container create f710690de26aa5c6301f196fafa6a43917f70a1e7d637a353183c80cb6d9e292 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-05-15T16:00:53.631 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:53 smithi053 podman[81874]: 2024-05-15 16:00:53.567100403 +0000 UTC m=+0.359029416 container init f710690de26aa5c6301f196fafa6a43917f70a1e7d637a353183c80cb6d9e292 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514) 2024-05-15T16:00:53.631 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:53 smithi053 podman[81874]: 2024-05-15 16:00:53.571074718 +0000 UTC m=+0.363003722 container start f710690de26aa5c6301f196fafa6a43917f70a1e7d637a353183c80cb6d9e292 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514) 2024-05-15T16:00:54.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:53 smithi043 ceph-mon[82531]: pgmap v208: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:00:54.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:53 smithi043 ceph-mon[82531]: osdmap e134: 8 total, 7 up, 8 in 2024-05-15T16:00:54.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:53 smithi043 ceph-mon[88099]: pgmap v208: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:00:54.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:53 smithi043 ceph-mon[88099]: osdmap e134: 8 total, 7 up, 8 in 2024-05-15T16:00:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:53 smithi053 ceph-mon[67220]: pgmap v208: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:00:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:53 smithi053 ceph-mon[67220]: osdmap e134: 8 total, 7 up, 8 in 2024-05-15T16:00:54.130 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:53 smithi053 podman[81874]: 2024-05-15 16:00:53.691696172 +0000 UTC m=+0.483625182 container attach f710690de26aa5c6301f196fafa6a43917f70a1e7d637a353183c80cb6d9e292 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-05-15T16:00:54.131 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:53 smithi053 podman[81874]: 2024-05-15 16:00:53.879432249 +0000 UTC m=+0.671361252 container died f710690de26aa5c6301f196fafa6a43917f70a1e7d637a353183c80cb6d9e292 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True) 2024-05-15T16:00:54.864 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:54 smithi053 podman[81900]: 2024-05-15 16:00:54.454161955 +0000 UTC m=+0.566524515 container remove f710690de26aa5c6301f196fafa6a43917f70a1e7d637a353183c80cb6d9e292 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240514, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:00:54.864 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:54 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.5.service: Deactivated successfully. 2024-05-15T16:00:54.864 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:54 smithi053 systemd[1]: Stopped Ceph osd.5 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:00:54.864 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:54 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.5.service: Consumed 17.566s CPU time. 2024-05-15T16:00:54.865 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:54 smithi053 systemd[1]: Starting Ceph osd.5 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:00:55.118 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:54 smithi053 ceph-mon[67220]: Health check failed: Degraded data redundancy: 33/627 objects degraded (5.263%), 7 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:55.118 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:54 smithi053 podman[82004]: 2024-05-15 16:00:54.897208314 +0000 UTC m=+0.096759586 container create 604b3da9ab7801a39f0975e1aa3d0a8714272ba46f8c04cf87f440ba07f231d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True) 2024-05-15T16:00:55.118 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:54 smithi053 podman[82004]: 2024-05-15 16:00:54.818246959 +0000 UTC m=+0.017798234 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:00:55.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:54 smithi043 ceph-mon[82531]: Health check failed: Degraded data redundancy: 33/627 objects degraded (5.263%), 7 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:55.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:54 smithi043 ceph-mon[88099]: Health check failed: Degraded data redundancy: 33/627 objects degraded (5.263%), 7 pgs degraded (PG_DEGRADED) 2024-05-15T16:00:55.380 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 podman[82004]: 2024-05-15 16:00:55.116825055 +0000 UTC m=+0.316376334 container init 604b3da9ab7801a39f0975e1aa3d0a8714272ba46f8c04cf87f440ba07f231d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-15T16:00:55.380 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 podman[82004]: 2024-05-15 16:00:55.120641093 +0000 UTC m=+0.320192369 container start 604b3da9ab7801a39f0975e1aa3d0a8714272ba46f8c04cf87f440ba07f231d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:00:55.380 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 podman[82004]: 2024-05-15 16:00:55.216614347 +0000 UTC m=+0.416165619 container attach 604b3da9ab7801a39f0975e1aa3d0a8714272ba46f8c04cf87f440ba07f231d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-05-15T16:00:55.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:55 smithi053 ceph-mon[67220]: pgmap v210: 161 pgs: 7 active+undersized+degraded, 11 stale+active+clean, 20 active+undersized, 123 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 33/627 objects degraded (5.263%) 2024-05-15T16:00:55.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate[82019]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:00:55.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate[82019]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:00:55.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 bash[82004]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:00:55.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 bash[82004]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:00:55.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 bash[82004]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-15T16:00:55.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 bash[82004]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-15T16:00:55.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 bash[82004]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-15T16:00:55.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 bash[82004]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-15T16:00:55.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 bash[82004]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-05-15T16:00:55.881 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 bash[82004]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-15T16:00:55.882 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 bash[82004]: --> ceph-volume raw activate successful for osd ID: 5 2024-05-15T16:00:55.882 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate[82019]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-15T16:00:55.882 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate[82019]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-15T16:00:55.882 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate[82019]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-15T16:00:55.882 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate[82019]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-15T16:00:55.882 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate[82019]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-05-15T16:00:55.882 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate[82019]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-15T16:00:55.882 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate[82019]: --> ceph-volume raw activate successful for osd ID: 5 2024-05-15T16:00:55.882 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:55 smithi053 podman[82004]: 2024-05-15 16:00:55.649950491 +0000 UTC m=+0.849501764 container died 604b3da9ab7801a39f0975e1aa3d0a8714272ba46f8c04cf87f440ba07f231d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-05-15T16:00:56.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:55 smithi043 ceph-mon[82531]: pgmap v210: 161 pgs: 7 active+undersized+degraded, 11 stale+active+clean, 20 active+undersized, 123 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 33/627 objects degraded (5.263%) 2024-05-15T16:00:56.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:55 smithi043 ceph-mon[88099]: pgmap v210: 161 pgs: 7 active+undersized+degraded, 11 stale+active+clean, 20 active+undersized, 123 active+clean; 585 KiB data, 193 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 33/627 objects degraded (5.263%) 2024-05-15T16:00:56.565 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:56 smithi053 podman[82151]: 2024-05-15 16:00:56.254532203 +0000 UTC m=+0.598261133 container remove 604b3da9ab7801a39f0975e1aa3d0a8714272ba46f8c04cf87f440ba07f231d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-05-15T16:00:56.565 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:56 smithi053 podman[82212]: 2024-05-15 16:00:56.472267229 +0000 UTC m=+0.106402302 container create f709584ed028a4fe85909b3c8253c158dd3ba0ab75652df13c215339b921f79e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0) 2024-05-15T16:00:56.565 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:56 smithi053 podman[82212]: 2024-05-15 16:00:56.384219792 +0000 UTC m=+0.018354854 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:00:56.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:00:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:00:56] "GET /metrics HTTP/1.1" 200 38257 "" "Prometheus/2.43.0" 2024-05-15T16:00:56.870 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:56 smithi053 podman[82212]: 2024-05-15 16:00:56.67480129 +0000 UTC m=+0.308936318 container init f709584ed028a4fe85909b3c8253c158dd3ba0ab75652df13c215339b921f79e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514) 2024-05-15T16:00:56.871 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:56 smithi053 podman[82212]: 2024-05-15 16:00:56.678592058 +0000 UTC m=+0.312727095 container start f709584ed028a4fe85909b3c8253c158dd3ba0ab75652df13c215339b921f79e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_CLEAN=True, ceph=True, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-15T16:00:56.871 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:56 smithi053 ceph-osd[82232]: -- 172.21.15.53:0/1904479064 <== mon.2 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55eb2a047680 con 0x55eb2a011000 2024-05-15T16:00:56.871 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:56 smithi053 bash[82212]: f709584ed028a4fe85909b3c8253c158dd3ba0ab75652df13c215339b921f79e 2024-05-15T16:00:56.871 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:56 smithi053 systemd[1]: Started Ceph osd.5 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:00:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:56 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:56 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:56 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:56 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:56 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:56 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:56 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:56 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:56 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:00:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:56 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:56 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:00:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:56 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:00:57.630 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:57 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:00:57.222+0000 7f76a014e7c0 -1 Falling back to public interface 2024-05-15T16:00:58.030 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:57 smithi053 ceph-mon[67220]: pgmap v211: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:00:58.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:57 smithi043 ceph-mon[82531]: pgmap v211: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:00:58.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:57 smithi043 ceph-mon[88099]: pgmap v211: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:01:00.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:00:59 smithi053 ceph-mon[67220]: pgmap v212: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:01:00.131 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:00:59.960+0000 7f76a014e7c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-05-15T16:01:00.131 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:00:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:00:59.987+0000 7f76a014e7c0 -1 osd.5 132 log_to_monitors true 2024-05-15T16:01:00.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:00:59 smithi043 ceph-mon[82531]: pgmap v212: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:01:00.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:00:59 smithi043 ceph-mon[88099]: pgmap v212: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:01:01.042 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:00 smithi043 ceph-mon[82531]: from='osd.5 [v2:172.21.15.53:6808/2697678971,v1:172.21.15.53:6809/2697678971]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T16:01:01.042 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:00 smithi043 ceph-mon[82531]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T16:01:01.042 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:00 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 71/627 objects degraded (11.324%), 19 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:01.043 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:00 smithi043 ceph-mon[88099]: from='osd.5 [v2:172.21.15.53:6808/2697678971,v1:172.21.15.53:6809/2697678971]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T16:01:01.043 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:00 smithi043 ceph-mon[88099]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T16:01:01.043 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:00 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 71/627 objects degraded (11.324%), 19 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:00 smithi053 ceph-mon[67220]: from='osd.5 [v2:172.21.15.53:6808/2697678971,v1:172.21.15.53:6809/2697678971]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T16:01:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:00 smithi053 ceph-mon[67220]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-15T16:01:01.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:00 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 71/627 objects degraded (11.324%), 19 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:01.630 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:01:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:01:01.169+0000 7f768d59d700 -1 osd.5 132 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T16:01:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:01 smithi053 ceph-mon[67220]: pgmap v213: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:01:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:01 smithi053 ceph-mon[67220]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-15T16:01:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:01 smithi053 ceph-mon[67220]: osdmap e135: 8 total, 7 up, 8 in 2024-05-15T16:01:02.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:01 smithi053 ceph-mon[67220]: from='osd.5 [v2:172.21.15.53:6808/2697678971,v1:172.21.15.53:6809/2697678971]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:02.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:01 smithi053 ceph-mon[67220]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:02.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:01 smithi053 ceph-mon[67220]: from='osd.5 ' entity='osd.5' 2024-05-15T16:01:02.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:01 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:02.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:01 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[82531]: pgmap v213: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:01:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[82531]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-15T16:01:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[82531]: osdmap e135: 8 total, 7 up, 8 in 2024-05-15T16:01:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[82531]: from='osd.5 [v2:172.21.15.53:6808/2697678971,v1:172.21.15.53:6809/2697678971]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[82531]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[82531]: from='osd.5 ' entity='osd.5' 2024-05-15T16:01:02.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:02.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:02.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[88099]: pgmap v213: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:01:02.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[88099]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-15T16:01:02.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[88099]: osdmap e135: 8 total, 7 up, 8 in 2024-05-15T16:01:02.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[88099]: from='osd.5 [v2:172.21.15.53:6808/2697678971,v1:172.21.15.53:6809/2697678971]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:02.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[88099]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:02.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[88099]: from='osd.5 ' entity='osd.5' 2024-05-15T16:01:02.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:02.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:01 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:03.241 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:02 smithi053 ceph-mon[67220]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:01:03.241 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:02 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T16:01:03.241 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:02 smithi053 ceph-mon[67220]: osd.5 [v2:172.21.15.53:6808/2697678971,v1:172.21.15.53:6809/2697678971] boot 2024-05-15T16:01:03.241 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:02 smithi053 ceph-mon[67220]: osdmap e136: 8 total, 8 up, 8 in 2024-05-15T16:01:03.241 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:02 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:03.241 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:02 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[82531]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:01:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T16:01:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[82531]: osd.5 [v2:172.21.15.53:6808/2697678971,v1:172.21.15.53:6809/2697678971] boot 2024-05-15T16:01:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[82531]: osdmap e136: 8 total, 8 up, 8 in 2024-05-15T16:01:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[88099]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:01:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-15T16:01:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[88099]: osd.5 [v2:172.21.15.53:6808/2697678971,v1:172.21.15.53:6809/2697678971] boot 2024-05-15T16:01:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[88099]: osdmap e136: 8 total, 8 up, 8 in 2024-05-15T16:01:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:02 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:04.513 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:04 smithi053 ceph-mon[67220]: pgmap v216: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:01:04.513 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:04 smithi053 ceph-mon[67220]: osdmap e137: 8 total, 8 up, 8 in 2024-05-15T16:01:04.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:04 smithi043 ceph-mon[82531]: pgmap v216: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:01:04.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:04 smithi043 ceph-mon[82531]: osdmap e137: 8 total, 8 up, 8 in 2024-05-15T16:01:04.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:04 smithi043 ceph-mon[88099]: pgmap v216: 161 pgs: 19 active+undersized+degraded, 40 active+undersized, 102 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-05-15T16:01:04.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:04 smithi043 ceph-mon[88099]: osdmap e137: 8 total, 8 up, 8 in 2024-05-15T16:01:05.973 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:05.973 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:05.973 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:05 smithi053 ceph-mon[67220]: pgmap v218: 161 pgs: 5 peering, 14 active+undersized+degraded, 28 active+undersized, 114 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 51/627 objects degraded (8.134%) 2024-05-15T16:01:06.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:06.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:06.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:05 smithi043 ceph-mon[82531]: pgmap v218: 161 pgs: 5 peering, 14 active+undersized+degraded, 28 active+undersized, 114 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 51/627 objects degraded (8.134%) 2024-05-15T16:01:06.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:06.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:06.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:05 smithi043 ceph-mon[88099]: pgmap v218: 161 pgs: 5 peering, 14 active+undersized+degraded, 28 active+undersized, 114 active+clean; 585 KiB data, 209 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 51/627 objects degraded (8.134%) 2024-05-15T16:01:06.648 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:01:06] "GET /metrics HTTP/1.1" 200 38115 "" "Prometheus/2.43.0" 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 51/627 objects degraded (8.134%), 14 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: pgmap v219: 161 pgs: 5 peering, 2 active+undersized+degraded, 4 active+undersized, 150 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 6/627 objects degraded (0.957%) 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: pgmap v220: 161 pgs: 5 peering, 2 active+undersized+degraded, 4 active+undersized, 150 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 6/627 objects degraded (0.957%) 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:01:06.916 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 51/627 objects degraded (8.134%), 14 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: pgmap v219: 161 pgs: 5 peering, 2 active+undersized+degraded, 4 active+undersized, 150 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 6/627 objects degraded (0.957%) 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: pgmap v220: 161 pgs: 5 peering, 2 active+undersized+degraded, 4 active+undersized, 150 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 6/627 objects degraded (0.957%) 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:06.917 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:01:06.918 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:06.918 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:06.918 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:06.918 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:06 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:01:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 51/627 objects degraded (8.134%), 14 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: pgmap v219: 161 pgs: 5 peering, 2 active+undersized+degraded, 4 active+undersized, 150 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 6/627 objects degraded (0.957%) 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: pgmap v220: 161 pgs: 5 peering, 2 active+undersized+degraded, 4 active+undersized, 150 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 6/627 objects degraded (0.957%) 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:06 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:01:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:08 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:01:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:08 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:08 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:08 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:08 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:08.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:08 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:01:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:08 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:08 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:08 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:08.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:08 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:08.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:01:07.998+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:08.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:08.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:01:08.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:08.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:08.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:08.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:08.092 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:08 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:01:08.012+0000 7fbf8c6f9700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-05-15T16:01:08.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:08 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:01:08.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:08 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:08.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:08 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:08.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:08 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:08.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:08 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:09.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:09 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:09.337 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:01:09.338 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:09.339 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:09.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:09 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:09 smithi043 ceph-mon[82531]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:09 smithi043 ceph-mon[82531]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:09 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.341 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.342 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:01:09.343 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:09.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:09.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:09.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:09.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:09.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.345 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:09 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:09.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:09 smithi043 ceph-mon[88099]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-05-15T16:01:09.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:09 smithi043 ceph-mon[88099]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:01:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:09 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:09.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:09.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:01:09.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:01:09.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:09 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:09 smithi053 ceph-mon[67220]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-05-15T16:01:09.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:09 smithi053 ceph-mon[67220]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:01:10.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:10 smithi043 ceph-mon[82531]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/627 objects degraded (0.957%), 2 pgs degraded) 2024-05-15T16:01:10.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:10 smithi043 ceph-mon[88099]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/627 objects degraded (0.957%), 2 pgs degraded) 2024-05-15T16:01:10.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:10 smithi053 ceph-mon[67220]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/627 objects degraded (0.957%), 2 pgs degraded) 2024-05-15T16:01:11.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:11 smithi043 ceph-mon[82531]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T16:01:11.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:11 smithi043 ceph-mon[88099]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T16:01:11.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:11 smithi053 ceph-mon[67220]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-05-15T16:01:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:13 smithi043 ceph-mon[82531]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-15T16:01:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:13 smithi043 ceph-mon[88099]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-15T16:01:13.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:13 smithi053 ceph-mon[67220]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-15T16:01:15.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:15 smithi043 ceph-mon[82531]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:01:15.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:15 smithi043 ceph-mon[88099]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:01:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:15 smithi053 ceph-mon[67220]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:01:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:16.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:01:16] "GET /metrics HTTP/1.1" 200 38115 "" "Prometheus/2.43.0" 2024-05-15T16:01:17.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:17 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:17.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:17 smithi043 ceph-mon[82531]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-15T16:01:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:17 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:17 smithi043 ceph-mon[88099]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-15T16:01:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:17 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:17 smithi053 ceph-mon[67220]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-15T16:01:19.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:19 smithi043 ceph-mon[82531]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:01:19.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:19 smithi043 ceph-mon[88099]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:01:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:19 smithi053 ceph-mon[67220]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:01:20.417 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T16:01:20.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:20 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:20.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:20.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:20 smithi043 ceph-mon[82531]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:01:20.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:20 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:20.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:20.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:20 smithi043 ceph-mon[88099]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (8m) 79s ago 30m 26.6M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (7m) 19s ago 30m 81.5M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 79s ago 29m 65.1M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (9m) 19s ago 35m 454M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (19m) 79s ago 36m 512M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (5m) 79s ago 36m 52.0M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (6m) 19s ago 35m 52.0M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (5m) 79s ago 35m 42.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (9m) 79s ago 31m 15.1M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T16:01:21.107 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (9m) 19s ago 31m 16.4M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (3m) 79s ago 34m 45.6M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (3m) 79s ago 34m 44.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (2m) 79s ago 33m 49.7M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (84s) 79s ago 33m 12.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 258344f90360 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (61s) 19s ago 33m 50.7M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 b342d0fab575 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (24s) 19s ago 32m 12.5M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 f709584ed028 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (32m) 19s ago 32m 58.8M 3249M 17.2.0 e1d6a67b021e 3ee49fe11313 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (31m) 19s ago 31m 62.1M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (8m) 19s ago 31m 73.3M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (29m) 79s ago 29m 78.6M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T16:01:21.108 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (29m) 19s ago 29m 77.4M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T16:01:21.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:20 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:21.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:21.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:20 smithi053 ceph-mon[67220]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:01:21.540 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:01:21.540 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T16:01:21.540 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T16:01:21.540 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:01:21.540 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T16:01:21.540 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 6 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 11 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T16:01:21.541 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:01:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:21 smithi043 ceph-mon[82531]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:21 smithi043 ceph-mon[82531]: from='client.54484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:21 smithi043 ceph-mon[82531]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:21.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:21 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2306378873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:21 smithi043 ceph-mon[88099]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:21.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:21 smithi043 ceph-mon[88099]: from='client.54484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:21.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:21 smithi043 ceph-mon[88099]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:21.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:21 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2306378873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:21.916 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:01:21.916 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T16:01:21.916 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T16:01:21.916 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-15T16:01:21.916 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T16:01:21.916 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "6/8 daemons upgraded", 2024-05-15T16:01:21.916 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading osd daemons", 2024-05-15T16:01:21.916 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T16:01:21.917 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:01:22.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:21 smithi053 ceph-mon[67220]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:22.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:21 smithi053 ceph-mon[67220]: from='client.54484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:22.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:21 smithi053 ceph-mon[67220]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:22.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:21 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2306378873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:23.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:22 smithi043 ceph-mon[82531]: from='client.54502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:23.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:22 smithi043 ceph-mon[82531]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:01:23.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:22 smithi043 ceph-mon[88099]: from='client.54502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:23.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:22 smithi043 ceph-mon[88099]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:01:23.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:22 smithi053 ceph-mon[67220]: from='client.54502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:23.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:22 smithi053 ceph-mon[67220]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:01:23.940 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:23.940 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:23 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:23.940 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:23 smithi053 ceph-mon[67220]: Upgrade: osd.6 is safe to restart 2024-05-15T16:01:24.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:24.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:23 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:24.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:23 smithi043 ceph-mon[82531]: Upgrade: osd.6 is safe to restart 2024-05-15T16:01:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:23 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-15T16:01:24.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:23 smithi043 ceph-mon[88099]: Upgrade: osd.6 is safe to restart 2024-05-15T16:01:25.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:24 smithi043 ceph-mon[82531]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:01:25.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:24 smithi043 ceph-mon[88099]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:01:25.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:24 smithi053 ceph-mon[67220]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:01:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:25 smithi053 ceph-mon[67220]: Upgrade: Updating osd.6 2024-05-15T16:01:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:25 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:25 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-15T16:01:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:25 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:26.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:25 smithi053 ceph-mon[67220]: Deploying daemon osd.6 on smithi053 2024-05-15T16:01:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:25 smithi043 ceph-mon[82531]: Upgrade: Updating osd.6 2024-05-15T16:01:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:25 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:25 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-15T16:01:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:25 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:25 smithi043 ceph-mon[82531]: Deploying daemon osd.6 on smithi053 2024-05-15T16:01:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:25 smithi043 ceph-mon[88099]: Upgrade: Updating osd.6 2024-05-15T16:01:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:25 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:25 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-15T16:01:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:25 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:26.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:25 smithi043 ceph-mon[88099]: Deploying daemon osd.6 on smithi053 2024-05-15T16:01:26.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:01:26] "GET /metrics HTTP/1.1" 200 38265 "" "Prometheus/2.43.0" 2024-05-15T16:01:27.304 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:26 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:27.304 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:26 smithi053 ceph-mon[67220]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:01:27.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:26 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:27.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:26 smithi043 ceph-mon[82531]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:01:27.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:26 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:27.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:26 smithi043 ceph-mon[88099]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:01:28.553 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:28 smithi053 systemd[1]: Stopping Ceph osd.6 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:01:28.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[34296]: 2024-05-15T16:01:28.549+0000 7fde34ed8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:01:28.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[34296]: 2024-05-15T16:01:28.549+0000 7fde34ed8700 -1 osd.6 137 *** Got signal Terminated *** 2024-05-15T16:01:28.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[34296]: 2024-05-15T16:01:28.549+0000 7fde34ed8700 -1 osd.6 137 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:01:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:29 smithi053 ceph-mon[67220]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:01:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:29 smithi053 ceph-mon[67220]: osd.6 marked itself down and dead 2024-05-15T16:01:29.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:29 smithi043 ceph-mon[82531]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:01:29.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:29 smithi043 ceph-mon[82531]: osd.6 marked itself down and dead 2024-05-15T16:01:29.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:29 smithi043 ceph-mon[88099]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:01:29.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:29 smithi043 ceph-mon[88099]: osd.6 marked itself down and dead 2024-05-15T16:01:29.937 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:29 smithi053 podman[86575]: 2024-05-15 16:01:29.655304025 +0000 UTC m=+1.220823754 container died 3ee49fe113137416d27d77c11631e07485eae8417d10810f2205d8b12819bef7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.openshift.tags=base centos centos-stream, distribution-scope=public, ceph=True, GIT_BRANCH=HEAD, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-15T16:01:30.320 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:30 smithi053 ceph-mon[67220]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:01:30.321 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:30 smithi053 ceph-mon[67220]: osdmap e138: 8 total, 7 up, 8 in 2024-05-15T16:01:30.321 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:30 smithi053 podman[86575]: 2024-05-15 16:01:30.084072371 +0000 UTC m=+1.649592100 container cleanup 3ee49fe113137416d27d77c11631e07485eae8417d10810f2205d8b12819bef7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, distribution-scope=public, version=8, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, release=754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-15T16:01:30.321 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:30 smithi053 bash[86575]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6 2024-05-15T16:01:30.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:30 smithi043 ceph-mon[82531]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:01:30.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:30 smithi043 ceph-mon[82531]: osdmap e138: 8 total, 7 up, 8 in 2024-05-15T16:01:30.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:30 smithi043 ceph-mon[88099]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:01:30.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:30 smithi043 ceph-mon[88099]: osdmap e138: 8 total, 7 up, 8 in 2024-05-15T16:01:30.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:30 smithi053 podman[86593]: 2024-05-15 16:01:30.319375822 +0000 UTC m=+0.656217752 container remove 3ee49fe113137416d27d77c11631e07485eae8417d10810f2205d8b12819bef7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6, io.openshift.tags=base centos centos-stream, release=754, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, name=centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870) 2024-05-15T16:01:31.083 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:30 smithi053 podman[86686]: 2024-05-15 16:01:30.687909168 +0000 UTC m=+0.018246466 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:01:31.083 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:30 smithi053 podman[86686]: 2024-05-15 16:01:30.787190349 +0000 UTC m=+0.117527644 container create 9964e5c22616a2f9140ad9be323f533185928d12cf7c8199cde810127002b998 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d) 2024-05-15T16:01:31.083 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:30 smithi053 podman[86686]: 2024-05-15 16:01:30.998222666 +0000 UTC m=+0.328559961 container init 9964e5c22616a2f9140ad9be323f533185928d12cf7c8199cde810127002b998 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, ceph=True) 2024-05-15T16:01:31.083 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:31 smithi053 podman[86686]: 2024-05-15 16:01:31.001873092 +0000 UTC m=+0.332210387 container start 9964e5c22616a2f9140ad9be323f533185928d12cf7c8199cde810127002b998 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0) 2024-05-15T16:01:31.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:31 smithi053 ceph-mon[67220]: pgmap v233: 161 pgs: 5 peering, 12 stale+active+clean, 144 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:01:31.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:31 smithi053 ceph-mon[67220]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-05-15T16:01:31.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:31 smithi053 ceph-mon[67220]: osdmap e139: 8 total, 7 up, 8 in 2024-05-15T16:01:31.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:31 smithi053 podman[86686]: 2024-05-15 16:01:31.081390609 +0000 UTC m=+0.411727912 container attach 9964e5c22616a2f9140ad9be323f533185928d12cf7c8199cde810127002b998 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-05-15T16:01:31.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:31 smithi053 podman[86686]: 2024-05-15 16:01:31.309513337 +0000 UTC m=+0.639850641 container died 9964e5c22616a2f9140ad9be323f533185928d12cf7c8199cde810127002b998 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T16:01:31.523 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:31 smithi043 ceph-mon[82531]: pgmap v233: 161 pgs: 5 peering, 12 stale+active+clean, 144 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:01:31.523 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:31 smithi043 ceph-mon[82531]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-05-15T16:01:31.523 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:31 smithi043 ceph-mon[82531]: osdmap e139: 8 total, 7 up, 8 in 2024-05-15T16:01:31.524 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:31 smithi043 ceph-mon[88099]: pgmap v233: 161 pgs: 5 peering, 12 stale+active+clean, 144 active+clean; 585 KiB data, 213 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:01:31.524 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:31 smithi043 ceph-mon[88099]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-05-15T16:01:31.524 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:31 smithi043 ceph-mon[88099]: osdmap e139: 8 total, 7 up, 8 in 2024-05-15T16:01:31.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:31 smithi053 podman[86711]: 2024-05-15 16:01:31.768942418 +0000 UTC m=+0.453162150 container remove 9964e5c22616a2f9140ad9be323f533185928d12cf7c8199cde810127002b998 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T16:01:31.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:31 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.6.service: Deactivated successfully. 2024-05-15T16:01:31.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:31 smithi053 systemd[1]: Stopped Ceph osd.6 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:01:31.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:31 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.6.service: Consumed 15.681s CPU time. 2024-05-15T16:01:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:32 smithi053 ceph-mon[67220]: Health check failed: Degraded data redundancy: 42/627 objects degraded (6.699%), 12 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:32.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:31 smithi053 systemd[1]: Starting Ceph osd.6 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:01:32.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:32 smithi053 podman[86811]: 2024-05-15 16:01:32.121674598 +0000 UTC m=+0.018466755 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:01:32.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:32 smithi053 podman[86811]: 2024-05-15 16:01:32.236805575 +0000 UTC m=+0.133597724 container create 50bfb242897676c20f56927c1262a6af5d44407a667adeade4457e51680fe6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-05-15T16:01:32.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:32 smithi043 ceph-mon[82531]: Health check failed: Degraded data redundancy: 42/627 objects degraded (6.699%), 12 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:32.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:32 smithi043 ceph-mon[88099]: Health check failed: Degraded data redundancy: 42/627 objects degraded (6.699%), 12 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:32.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:32 smithi053 podman[86811]: 2024-05-15 16:01:32.522786489 +0000 UTC m=+0.419578629 container init 50bfb242897676c20f56927c1262a6af5d44407a667adeade4457e51680fe6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514) 2024-05-15T16:01:32.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:32 smithi053 podman[86811]: 2024-05-15 16:01:32.526538291 +0000 UTC m=+0.423330445 container start 50bfb242897676c20f56927c1262a6af5d44407a667adeade4457e51680fe6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-05-15T16:01:32.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:32 smithi053 podman[86811]: 2024-05-15 16:01:32.597641714 +0000 UTC m=+0.494433864 container attach 50bfb242897676c20f56927c1262a6af5d44407a667adeade4457e51680fe6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD) 2024-05-15T16:01:33.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:33 smithi053 ceph-mon[67220]: pgmap v235: 161 pgs: 12 active+undersized+degraded, 5 peering, 8 stale+active+clean, 18 active+undersized, 118 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 42/627 objects degraded (6.699%) 2024-05-15T16:01:33.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate[86826]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:01:33.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate[86826]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 bash[86811]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 bash[86811]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 bash[86811]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 bash[86811]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 bash[86811]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 bash[86811]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 bash[86811]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 bash[86811]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 bash[86811]: --> ceph-volume raw activate successful for osd ID: 6 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate[86826]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate[86826]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate[86826]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate[86826]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate[86826]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate[86826]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-15T16:01:33.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate[86826]: --> ceph-volume raw activate successful for osd ID: 6 2024-05-15T16:01:33.382 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 podman[86811]: 2024-05-15 16:01:33.06006125 +0000 UTC m=+0.956853400 container died 50bfb242897676c20f56927c1262a6af5d44407a667adeade4457e51680fe6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-05-15T16:01:33.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:33 smithi043 ceph-mon[82531]: pgmap v235: 161 pgs: 12 active+undersized+degraded, 5 peering, 8 stale+active+clean, 18 active+undersized, 118 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 42/627 objects degraded (6.699%) 2024-05-15T16:01:33.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:33 smithi043 ceph-mon[88099]: pgmap v235: 161 pgs: 12 active+undersized+degraded, 5 peering, 8 stale+active+clean, 18 active+undersized, 118 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 42/627 objects degraded (6.699%) 2024-05-15T16:01:33.843 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 podman[86954]: 2024-05-15 16:01:33.619070313 +0000 UTC m=+0.552186017 container remove 50bfb242897676c20f56927c1262a6af5d44407a667adeade4457e51680fe6f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=) 2024-05-15T16:01:34.116 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 podman[87015]: 2024-05-15 16:01:33.745700215 +0000 UTC m=+0.018446508 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:01:34.116 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:33 smithi053 podman[87015]: 2024-05-15 16:01:33.861808164 +0000 UTC m=+0.134554447 container create d5353ef723114a7838c03c46283d6f5fa8c2d8c4c7ae5ee0024d3ad3f3e82664 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5) 2024-05-15T16:01:34.116 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:34 smithi053 podman[87015]: 2024-05-15 16:01:34.115127774 +0000 UTC m=+0.387874057 container init d5353ef723114a7838c03c46283d6f5fa8c2d8c4c7ae5ee0024d3ad3f3e82664 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:01:34.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:34 smithi053 podman[87015]: 2024-05-15 16:01:34.118574653 +0000 UTC m=+0.391320937 container start d5353ef723114a7838c03c46283d6f5fa8c2d8c4c7ae5ee0024d3ad3f3e82664 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514) 2024-05-15T16:01:34.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:34 smithi053 ceph-osd[87033]: -- 172.21.15.53:0/133863588 <== mon.2 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558a0fd85680 con 0x558a0fd4f400 2024-05-15T16:01:34.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:34 smithi053 bash[87015]: d5353ef723114a7838c03c46283d6f5fa8c2d8c4c7ae5ee0024d3ad3f3e82664 2024-05-15T16:01:34.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:34 smithi053 systemd[1]: Started Ceph osd.6 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:01:35.122 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:01:34.661+0000 7fb53b01b7c0 -1 Falling back to public interface 2024-05-15T16:01:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:35 smithi053 ceph-mon[67220]: pgmap v236: 161 pgs: 17 active+undersized+degraded, 5 peering, 31 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 B/s wr, 2 op/s; 67/627 objects degraded (10.686%) 2024-05-15T16:01:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:01:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[88099]: pgmap v236: 161 pgs: 17 active+undersized+degraded, 5 peering, 31 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 B/s wr, 2 op/s; 67/627 objects degraded (10.686%) 2024-05-15T16:01:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:01:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:35.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[82531]: pgmap v236: 161 pgs: 17 active+undersized+degraded, 5 peering, 31 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 B/s wr, 2 op/s; 67/627 objects degraded (10.686%) 2024-05-15T16:01:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:01:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:35.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:36.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:36 smithi043 ceph-mon[82531]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-05-15T16:01:36.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:36 smithi043 ceph-mon[88099]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-05-15T16:01:36.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:01:36] "GET /metrics HTTP/1.1" 200 38275 "" "Prometheus/2.43.0" 2024-05-15T16:01:36.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:36 smithi053 ceph-mon[67220]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-05-15T16:01:37.585 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:37 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:37.585 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:37 smithi053 ceph-mon[67220]: pgmap v237: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 48 op/s; 76/627 objects degraded (12.121%) 2024-05-15T16:01:37.585 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:01:37.393+0000 7fb53b01b7c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-05-15T16:01:37.585 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:01:37.421+0000 7fb53b01b7c0 -1 osd.6 137 log_to_monitors true 2024-05-15T16:01:37.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:37 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:37.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:37 smithi043 ceph-mon[82531]: pgmap v237: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 48 op/s; 76/627 objects degraded (12.121%) 2024-05-15T16:01:37.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:37 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:37.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:37 smithi043 ceph-mon[88099]: pgmap v237: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 48 op/s; 76/627 objects degraded (12.121%) 2024-05-15T16:01:38.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:38 smithi043 ceph-mon[88099]: from='osd.6 [v2:172.21.15.53:6816/2211290854,v1:172.21.15.53:6817/2211290854]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T16:01:38.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:38 smithi043 ceph-mon[88099]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T16:01:38.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:38 smithi043 ceph-mon[82531]: from='osd.6 [v2:172.21.15.53:6816/2211290854,v1:172.21.15.53:6817/2211290854]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T16:01:38.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:38 smithi043 ceph-mon[82531]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T16:01:38.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:38 smithi053 ceph-mon[67220]: from='osd.6 [v2:172.21.15.53:6816/2211290854,v1:172.21.15.53:6817/2211290854]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T16:01:38.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:38 smithi053 ceph-mon[67220]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-15T16:01:38.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:01:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:01:38.350+0000 7fb52c472700 -1 osd.6 137 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T16:01:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:39 smithi053 ceph-mon[67220]: pgmap v238: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 0 B/s wr, 95 op/s; 76/627 objects degraded (12.121%) 2024-05-15T16:01:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:39 smithi053 ceph-mon[67220]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-15T16:01:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:39 smithi053 ceph-mon[67220]: from='osd.6 [v2:172.21.15.53:6816/2211290854,v1:172.21.15.53:6817/2211290854]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:39.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:39 smithi053 ceph-mon[67220]: osdmap e140: 8 total, 7 up, 8 in 2024-05-15T16:01:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:39 smithi053 ceph-mon[67220]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:39 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:39.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:39 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[88099]: pgmap v238: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 0 B/s wr, 95 op/s; 76/627 objects degraded (12.121%) 2024-05-15T16:01:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[88099]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-15T16:01:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[88099]: from='osd.6 [v2:172.21.15.53:6816/2211290854,v1:172.21.15.53:6817/2211290854]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[88099]: osdmap e140: 8 total, 7 up, 8 in 2024-05-15T16:01:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[88099]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:39.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:39.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[82531]: pgmap v238: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 0 B/s wr, 95 op/s; 76/627 objects degraded (12.121%) 2024-05-15T16:01:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[82531]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-15T16:01:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[82531]: from='osd.6 [v2:172.21.15.53:6816/2211290854,v1:172.21.15.53:6817/2211290854]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[82531]: osdmap e140: 8 total, 7 up, 8 in 2024-05-15T16:01:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[82531]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:01:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:39.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:39 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:40.613 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:40 smithi053 ceph-mon[67220]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:01:40.613 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:40 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T16:01:40.614 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:40 smithi053 ceph-mon[67220]: osd.6 [v2:172.21.15.53:6816/2211290854,v1:172.21.15.53:6817/2211290854] boot 2024-05-15T16:01:40.614 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:40 smithi053 ceph-mon[67220]: osdmap e141: 8 total, 8 up, 8 in 2024-05-15T16:01:40.614 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:40 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:40.614 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:40 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:40.614 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:40 smithi053 ceph-mon[67220]: osdmap e142: 8 total, 8 up, 8 in 2024-05-15T16:01:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[88099]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:01:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T16:01:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[88099]: osd.6 [v2:172.21.15.53:6816/2211290854,v1:172.21.15.53:6817/2211290854] boot 2024-05-15T16:01:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[88099]: osdmap e141: 8 total, 8 up, 8 in 2024-05-15T16:01:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:40.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[88099]: osdmap e142: 8 total, 8 up, 8 in 2024-05-15T16:01:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[82531]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:01:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-15T16:01:40.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[82531]: osd.6 [v2:172.21.15.53:6816/2211290854,v1:172.21.15.53:6817/2211290854] boot 2024-05-15T16:01:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[82531]: osdmap e141: 8 total, 8 up, 8 in 2024-05-15T16:01:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:40.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:40 smithi043 ceph-mon[82531]: osdmap e142: 8 total, 8 up, 8 in 2024-05-15T16:01:41.617 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:41 smithi053 ceph-mon[67220]: pgmap v241: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 145 op/s; 76/627 objects degraded (12.121%) 2024-05-15T16:01:41.617 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:41 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:41 smithi043 ceph-mon[82531]: pgmap v241: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 145 op/s; 76/627 objects degraded (12.121%) 2024-05-15T16:01:41.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:41 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:41 smithi043 ceph-mon[88099]: pgmap v241: 161 pgs: 18 active+undersized+degraded, 35 active+undersized, 108 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 145 op/s; 76/627 objects degraded (12.121%) 2024-05-15T16:01:41.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:41 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:42 smithi053 ceph-mon[67220]: pgmap v243: 161 pgs: 1 active+recovering+degraded, 8 active+undersized+degraded, 1 active+recovery_wait+degraded, 19 active+undersized, 132 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 0 B/s wr, 169 op/s; 43/627 objects degraded (6.858%); 0 B/s, 3 objects/s recovering 2024-05-15T16:01:42.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:42 smithi043 ceph-mon[88099]: pgmap v243: 161 pgs: 1 active+recovering+degraded, 8 active+undersized+degraded, 1 active+recovery_wait+degraded, 19 active+undersized, 132 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 0 B/s wr, 169 op/s; 43/627 objects degraded (6.858%); 0 B/s, 3 objects/s recovering 2024-05-15T16:01:42.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:42 smithi043 ceph-mon[82531]: pgmap v243: 161 pgs: 1 active+recovering+degraded, 8 active+undersized+degraded, 1 active+recovery_wait+degraded, 19 active+undersized, 132 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 0 B/s wr, 169 op/s; 43/627 objects degraded (6.858%); 0 B/s, 3 objects/s recovering 2024-05-15T16:01:44.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:44.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:44.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:44.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:01:44.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: pgmap v244: 161 pgs: 1 active+recovering+degraded, 4 active+undersized+degraded, 1 active+recovery_wait+degraded, 9 active+undersized, 146 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 0 B/s wr, 103 op/s; 29/627 objects degraded (4.625%); 0 B/s, 6 objects/s recovering 2024-05-15T16:01:44.403 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: pgmap v244: 161 pgs: 1 active+recovering+degraded, 4 active+undersized+degraded, 1 active+recovery_wait+degraded, 9 active+undersized, 146 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 0 B/s wr, 103 op/s; 29/627 objects degraded (4.625%); 0 B/s, 6 objects/s recovering 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:44.404 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:01:44.405 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:44.405 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:44.405 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:44.405 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:44 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:01:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:01:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: pgmap v244: 161 pgs: 1 active+recovering+degraded, 4 active+undersized+degraded, 1 active+recovery_wait+degraded, 9 active+undersized, 146 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 0 B/s wr, 103 op/s; 29/627 objects degraded (4.625%); 0 B/s, 6 objects/s recovering 2024-05-15T16:01:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:01:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:01:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:01:44.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:44 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:01:45.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:45.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:01:45.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:45.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:45.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:45.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:01:45.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:45.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:01:45.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:45.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:45.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:45.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:01:45.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:45.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:45.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:45.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:45.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:45.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:45.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.596 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:01:45.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:45.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:45.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:45.596 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:45 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:01:45.335+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:45.597 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:01:45.598 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:45.599 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:45.600 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:45.601 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.601 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.601 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:45 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:01:45.349+0000 7fbf8c6f9700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-05-15T16:01:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:45 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:01:45.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:01:45.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:45.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:45 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:01:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:46 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:01:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:46 smithi053 ceph-mon[67220]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-05-15T16:01:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:46 smithi053 ceph-mon[67220]: pgmap v245: 161 pgs: 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 159 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 78 op/s; 6/627 objects degraded (0.957%); 0 B/s, 6 objects/s recovering 2024-05-15T16:01:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:46 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:46 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:46 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:01:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:46 smithi043 ceph-mon[82531]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-05-15T16:01:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:46 smithi043 ceph-mon[82531]: pgmap v245: 161 pgs: 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 159 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 78 op/s; 6/627 objects degraded (0.957%); 0 B/s, 6 objects/s recovering 2024-05-15T16:01:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:46 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:46.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:46 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:46.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:01:46] "GET /metrics HTTP/1.1" 200 38275 "" "Prometheus/2.43.0" 2024-05-15T16:01:46.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:46 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:01:46.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:46 smithi043 ceph-mon[88099]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-05-15T16:01:46.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:46 smithi043 ceph-mon[88099]: pgmap v245: 161 pgs: 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 159 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 78 op/s; 6/627 objects degraded (0.957%); 0 B/s, 6 objects/s recovering 2024-05-15T16:01:46.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:46 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 6 pgs degraded (PG_DEGRADED) 2024-05-15T16:01:46.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:46 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:47.799 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:47 smithi053 ceph-mon[67220]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/627 objects degraded (0.957%), 2 pgs degraded) 2024-05-15T16:01:47.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:47 smithi043 ceph-mon[82531]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/627 objects degraded (0.957%), 2 pgs degraded) 2024-05-15T16:01:47.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:47 smithi043 ceph-mon[88099]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/627 objects degraded (0.957%), 2 pgs degraded) 2024-05-15T16:01:48.754 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:48 smithi053 ceph-mon[67220]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 34 op/s; 0 B/s, 6 objects/s recovering 2024-05-15T16:01:48.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:48 smithi043 ceph-mon[82531]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 34 op/s; 0 B/s, 6 objects/s recovering 2024-05-15T16:01:48.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:48 smithi043 ceph-mon[88099]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 34 op/s; 0 B/s, 6 objects/s recovering 2024-05-15T16:01:51.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:50 smithi043 ceph-mon[82531]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 29 op/s; 0 B/s, 5 objects/s recovering 2024-05-15T16:01:51.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:50 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:51.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:50 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:51.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:50 smithi043 ceph-mon[88099]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 29 op/s; 0 B/s, 5 objects/s recovering 2024-05-15T16:01:51.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:50 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:51.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:50 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:51.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:50 smithi053 ceph-mon[67220]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 29 op/s; 0 B/s, 5 objects/s recovering 2024-05-15T16:01:51.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:50 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:51.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:50 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:01:52.302 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T16:01:52.994 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (8m) 110s ago 31m 26.6M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (8m) 14s ago 30m 82.8M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 110s ago 30m 65.1M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (10m) 14s ago 35m 455M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (20m) 110s ago 37m 512M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (6m) 110s ago 37m 52.0M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (7m) 14s ago 35m 52.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (6m) 110s ago 35m 42.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (9m) 110s ago 31m 15.1M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (9m) 14s ago 31m 16.2M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (4m) 110s ago 35m 45.6M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (3m) 110s ago 34m 44.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (2m) 110s ago 34m 49.7M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T16:01:52.995 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (115s) 110s ago 34m 12.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 258344f90360 2024-05-15T16:01:52.996 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (93s) 14s ago 33m 52.7M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 b342d0fab575 2024-05-15T16:01:52.996 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (56s) 14s ago 33m 52.1M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 f709584ed028 2024-05-15T16:01:52.996 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (19s) 14s ago 32m 12.6M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 d5353ef72311 2024-05-15T16:01:52.996 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (32m) 14s ago 32m 63.3M 3249M 17.2.0 e1d6a67b021e 10212d144623 2024-05-15T16:01:52.996 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (9m) 14s ago 31m 76.6M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T16:01:52.996 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (30m) 110s ago 30m 78.6M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T16:01:52.996 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (30m) 14s ago 30m 77.6M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T16:01:52.996 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:52 smithi043 ceph-mon[82531]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 25 op/s; 0 B/s, 4 objects/s recovering 2024-05-15T16:01:52.996 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:52 smithi043 ceph-mon[82531]: from='client.44574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:52.997 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:52 smithi043 ceph-mon[88099]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 25 op/s; 0 B/s, 4 objects/s recovering 2024-05-15T16:01:52.997 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:52 smithi043 ceph-mon[88099]: from='client.44574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:53.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:52 smithi053 ceph-mon[67220]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 25 op/s; 0 B/s, 4 objects/s recovering 2024-05-15T16:01:53.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:52 smithi053 ceph-mon[67220]: from='client.44574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:53.410 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 7 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T16:01:53.411 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:01:53.412 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T16:01:53.412 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-15T16:01:53.412 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 12 2024-05-15T16:01:53.412 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T16:01:53.412 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:01:53.754 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:01:53.754 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T16:01:53.755 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T16:01:53.755 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-05-15T16:01:53.755 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T16:01:53.755 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "7/8 daemons upgraded", 2024-05-15T16:01:53.755 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading osd daemons", 2024-05-15T16:01:53.755 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T16:01:53.755 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:01:53.755 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:53 smithi043 ceph-mon[82531]: from='client.54517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:53.755 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:53 smithi043 ceph-mon[82531]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:53.755 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:53 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2803376410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:53.756 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:53 smithi043 ceph-mon[88099]: from='client.54517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:53.756 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:53 smithi043 ceph-mon[88099]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:53.756 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:53 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2803376410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:53 smithi053 ceph-mon[67220]: from='client.54517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:53 smithi053 ceph-mon[67220]: from='client.44586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:54.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:53 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2803376410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:01:55.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:54 smithi043 ceph-mon[82531]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s; 0 B/s, 2 objects/s recovering 2024-05-15T16:01:55.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:54 smithi043 ceph-mon[82531]: from='client.44598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:55.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:54 smithi043 ceph-mon[88099]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s; 0 B/s, 2 objects/s recovering 2024-05-15T16:01:55.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:54 smithi043 ceph-mon[88099]: from='client.44598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:55.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:54 smithi053 ceph-mon[67220]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s; 0 B/s, 2 objects/s recovering 2024-05-15T16:01:55.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:54 smithi053 ceph-mon[67220]: from='client.44598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:01:56.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:55 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:56.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:55 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:56.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:55 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:01:56.813 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:01:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:01:56] "GET /metrics HTTP/1.1" 200 38286 "" "Prometheus/2.43.0" 2024-05-15T16:01:57.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:56 smithi043 ceph-mon[82531]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s; 0 B/s, 1 objects/s recovering 2024-05-15T16:01:57.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:56 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:57.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:56 smithi043 ceph-mon[88099]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s; 0 B/s, 1 objects/s recovering 2024-05-15T16:01:57.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:56 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:56 smithi053 ceph-mon[67220]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s; 0 B/s, 1 objects/s recovering 2024-05-15T16:01:57.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:56 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:01:59.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:01:58 smithi043 ceph-mon[82531]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-05-15T16:01:59.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:01:58 smithi043 ceph-mon[88099]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-05-15T16:01:59.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:01:58 smithi053 ceph-mon[67220]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-05-15T16:02:01.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:00 smithi043 ceph-mon[82531]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T16:02:01.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:00 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:02:01.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:00 smithi043 ceph-mon[88099]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T16:02:01.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:00 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:02:01.103 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:00 smithi053 ceph-mon[67220]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T16:02:01.103 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:00 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:02:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:01 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:02:02.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:01 smithi043 ceph-mon[82531]: Upgrade: osd.7 is safe to restart 2024-05-15T16:02:02.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:01 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:02:02.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:01 smithi043 ceph-mon[88099]: Upgrade: osd.7 is safe to restart 2024-05-15T16:02:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:01 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-15T16:02:02.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:01 smithi053 ceph-mon[67220]: Upgrade: osd.7 is safe to restart 2024-05-15T16:02:03.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:02 smithi053 ceph-mon[67220]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:03.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:02 smithi053 ceph-mon[67220]: Upgrade: Updating osd.7 2024-05-15T16:02:03.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:02 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:03.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:02 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-15T16:02:03.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:02 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:03.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:02 smithi053 ceph-mon[67220]: Deploying daemon osd.7 on smithi053 2024-05-15T16:02:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[82531]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[82531]: Upgrade: Updating osd.7 2024-05-15T16:02:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-15T16:02:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[82531]: Deploying daemon osd.7 on smithi053 2024-05-15T16:02:03.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[88099]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[88099]: Upgrade: Updating osd.7 2024-05-15T16:02:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-15T16:02:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:03.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:02 smithi043 ceph-mon[88099]: Deploying daemon osd.7 on smithi053 2024-05-15T16:02:05.274 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:04 smithi053 ceph-mon[67220]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:05.274 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:05.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:04 smithi043 ceph-mon[82531]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:05.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:05.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:04 smithi043 ceph-mon[88099]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:05.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:05.558 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:05 smithi053 systemd[1]: Stopping Ceph osd.7 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:02:05.880 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:05 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[37268]: 2024-05-15T16:02:05.555+0000 7fa054948700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:02:05.880 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:05 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[37268]: 2024-05-15T16:02:05.555+0000 7fa054948700 -1 osd.7 142 *** Got signal Terminated *** 2024-05-15T16:02:05.880 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:05 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[37268]: 2024-05-15T16:02:05.555+0000 7fa054948700 -1 osd.7 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:02:06.252 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:05 smithi053 ceph-mon[67220]: osd.7 marked itself down and dead 2024-05-15T16:02:06.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:05 smithi043 ceph-mon[82531]: osd.7 marked itself down and dead 2024-05-15T16:02:06.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:05 smithi043 ceph-mon[88099]: osd.7 marked itself down and dead 2024-05-15T16:02:06.517 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:06 smithi053 podman[91523]: 2024-05-15 16:02:06.25159057 +0000 UTC m=+0.821480156 container died 10212d14462364494f1c3979b29019986ecc2f1ce229738fdd9bea9bfa7bd876 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7, ceph=True, vendor=Red Hat, Inc., GIT_CLEAN=True, io.openshift.expose-services=, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream) 2024-05-15T16:02:06.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:02:06] "GET /metrics HTTP/1.1" 200 38285 "" "Prometheus/2.43.0" 2024-05-15T16:02:06.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:06 smithi053 podman[91523]: 2024-05-15 16:02:06.606267696 +0000 UTC m=+1.176157281 container cleanup 10212d14462364494f1c3979b29019986ecc2f1ce229738fdd9bea9bfa7bd876 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_CLEAN=True, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container) 2024-05-15T16:02:06.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:06 smithi053 bash[91523]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7 2024-05-15T16:02:07.130 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:06 smithi053 podman[91540]: 2024-05-15 16:02:06.865738385 +0000 UTC m=+0.605581021 container remove 10212d14462364494f1c3979b29019986ecc2f1ce229738fdd9bea9bfa7bd876 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, architecture=x86_64, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-type=git, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-15T16:02:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:06 smithi053 ceph-mon[67220]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:06 smithi053 ceph-mon[67220]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:02:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:06 smithi053 ceph-mon[67220]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-15T16:02:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:06 smithi053 ceph-mon[67220]: osdmap e143: 8 total, 7 up, 8 in 2024-05-15T16:02:07.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:06 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:06 smithi043 ceph-mon[82531]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:06 smithi043 ceph-mon[82531]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:02:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:06 smithi043 ceph-mon[82531]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-15T16:02:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:06 smithi043 ceph-mon[82531]: osdmap e143: 8 total, 7 up, 8 in 2024-05-15T16:02:07.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:06 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:06 smithi043 ceph-mon[88099]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:06 smithi043 ceph-mon[88099]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-15T16:02:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:06 smithi043 ceph-mon[88099]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-15T16:02:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:06 smithi043 ceph-mon[88099]: osdmap e143: 8 total, 7 up, 8 in 2024-05-15T16:02:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:06 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:07.479 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:07 smithi053 podman[91631]: 2024-05-15 16:02:07.113464098 +0000 UTC m=+0.018056819 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:02:07.479 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:07 smithi053 podman[91631]: 2024-05-15 16:02:07.216958143 +0000 UTC m=+0.121550845 container create 73ef50385c831c674c37b44879bd34aa88bfa4f10571061f2a2b71fa99ae056a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True) 2024-05-15T16:02:07.789 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:07 smithi053 podman[91631]: 2024-05-15 16:02:07.478320548 +0000 UTC m=+0.382913264 container init 73ef50385c831c674c37b44879bd34aa88bfa4f10571061f2a2b71fa99ae056a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, ceph=True) 2024-05-15T16:02:07.789 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:07 smithi053 podman[91631]: 2024-05-15 16:02:07.481723794 +0000 UTC m=+0.386316504 container start 73ef50385c831c674c37b44879bd34aa88bfa4f10571061f2a2b71fa99ae056a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:02:07.789 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:07 smithi053 podman[91631]: 2024-05-15 16:02:07.544900277 +0000 UTC m=+0.449492978 container attach 73ef50385c831c674c37b44879bd34aa88bfa4f10571061f2a2b71fa99ae056a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:02:08.092 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:07 smithi053 ceph-mon[67220]: osdmap e144: 8 total, 7 up, 8 in 2024-05-15T16:02:08.092 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:07 smithi053 podman[91631]: 2024-05-15 16:02:07.788102034 +0000 UTC m=+0.692694735 container died 73ef50385c831c674c37b44879bd34aa88bfa4f10571061f2a2b71fa99ae056a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS) 2024-05-15T16:02:08.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:07 smithi043 ceph-mon[82531]: osdmap e144: 8 total, 7 up, 8 in 2024-05-15T16:02:08.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:07 smithi043 ceph-mon[88099]: osdmap e144: 8 total, 7 up, 8 in 2024-05-15T16:02:08.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:08 smithi053 podman[91656]: 2024-05-15 16:02:08.340911624 +0000 UTC m=+0.546159458 container remove 73ef50385c831c674c37b44879bd34aa88bfa4f10571061f2a2b71fa99ae056a (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-05-15T16:02:08.631 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:08 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.7.service: Deactivated successfully. 2024-05-15T16:02:08.631 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:08 smithi053 systemd[1]: Stopped Ceph osd.7 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:02:08.631 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:08 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.7.service: Consumed 19.013s CPU time. 2024-05-15T16:02:08.631 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:08 smithi053 systemd[1]: Starting Ceph osd.7 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:02:09.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:08 smithi053 ceph-mon[67220]: pgmap v258: 161 pgs: 22 peering, 19 stale+active+clean, 120 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:02:09.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:08 smithi053 ceph-mon[67220]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-05-15T16:02:09.130 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:08 smithi053 podman[91757]: 2024-05-15 16:02:08.792095376 +0000 UTC m=+0.110718492 container create 202c806eba3f73c00f3babc32527c7cfef035b0beb22fd70b93b4fd7e14fbc81 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD) 2024-05-15T16:02:09.130 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:08 smithi053 podman[91757]: 2024-05-15 16:02:08.699272691 +0000 UTC m=+0.017895248 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:02:09.131 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 podman[91757]: 2024-05-15 16:02:09.027949252 +0000 UTC m=+0.346571797 container init 202c806eba3f73c00f3babc32527c7cfef035b0beb22fd70b93b4fd7e14fbc81 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:02:09.131 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 podman[91757]: 2024-05-15 16:02:09.031339247 +0000 UTC m=+0.349961793 container start 202c806eba3f73c00f3babc32527c7cfef035b0beb22fd70b93b4fd7e14fbc81 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-05-15T16:02:09.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:08 smithi043 ceph-mon[82531]: pgmap v258: 161 pgs: 22 peering, 19 stale+active+clean, 120 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:02:09.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:08 smithi043 ceph-mon[82531]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-05-15T16:02:09.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:08 smithi043 ceph-mon[88099]: pgmap v258: 161 pgs: 22 peering, 19 stale+active+clean, 120 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:02:09.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:08 smithi043 ceph-mon[88099]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-05-15T16:02:09.534 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 podman[91757]: 2024-05-15 16:02:09.161597135 +0000 UTC m=+0.480219694 container attach 202c806eba3f73c00f3babc32527c7cfef035b0beb22fd70b93b4fd7e14fbc81 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True) 2024-05-15T16:02:09.866 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate[91771]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate[91771]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 bash[91757]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 bash[91757]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 bash[91757]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 bash[91757]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 bash[91757]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 bash[91757]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 bash[91757]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 bash[91757]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 bash[91757]: --> ceph-volume raw activate successful for osd ID: 7 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate[91771]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate[91771]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-15T16:02:09.867 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate[91771]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-15T16:02:09.868 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate[91771]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-15T16:02:09.868 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate[91771]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-05-15T16:02:09.868 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate[91771]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-15T16:02:09.868 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate[91771]: --> ceph-volume raw activate successful for osd ID: 7 2024-05-15T16:02:09.868 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:09 smithi053 podman[91757]: 2024-05-15 16:02:09.557751746 +0000 UTC m=+0.876374301 container died 202c806eba3f73c00f3babc32527c7cfef035b0beb22fd70b93b4fd7e14fbc81 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-05-15T16:02:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:09 smithi053 ceph-mon[67220]: Health check failed: Degraded data redundancy: 48/627 objects degraded (7.656%), 16 pgs degraded (PG_DEGRADED) 2024-05-15T16:02:10.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:09 smithi043 ceph-mon[82531]: Health check failed: Degraded data redundancy: 48/627 objects degraded (7.656%), 16 pgs degraded (PG_DEGRADED) 2024-05-15T16:02:10.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:09 smithi043 ceph-mon[88099]: Health check failed: Degraded data redundancy: 48/627 objects degraded (7.656%), 16 pgs degraded (PG_DEGRADED) 2024-05-15T16:02:10.476 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:10 smithi053 podman[91894]: 2024-05-15 16:02:10.140948941 +0000 UTC m=+0.577229248 container remove 202c806eba3f73c00f3babc32527c7cfef035b0beb22fd70b93b4fd7e14fbc81 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-15T16:02:10.476 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:10 smithi053 podman[91955]: 2024-05-15 16:02:10.358788654 +0000 UTC m=+0.112473453 container create 784f18f73ebde8c67b43763c8aed670eea1f4c3dcba305550ba4cfc66e3dc9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-05-15T16:02:10.476 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:10 smithi053 podman[91955]: 2024-05-15 16:02:10.264543545 +0000 UTC m=+0.018228371 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:02:10.880 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:10 smithi053 podman[91955]: 2024-05-15 16:02:10.602875574 +0000 UTC m=+0.356560382 container init 784f18f73ebde8c67b43763c8aed670eea1f4c3dcba305550ba4cfc66e3dc9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240514) 2024-05-15T16:02:10.880 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:10 smithi053 podman[91955]: 2024-05-15 16:02:10.60678117 +0000 UTC m=+0.360465975 container start 784f18f73ebde8c67b43763c8aed670eea1f4c3dcba305550ba4cfc66e3dc9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:02:10.881 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:10 smithi053 ceph-osd[91974]: -- 172.21.15.53:0/644686977 <== mon.2 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5591b74a3680 con 0x5591b746d400 2024-05-15T16:02:10.881 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:10 smithi053 bash[91955]: 784f18f73ebde8c67b43763c8aed670eea1f4c3dcba305550ba4cfc66e3dc9df 2024-05-15T16:02:10.881 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:10 smithi053 systemd[1]: Started Ceph osd.7 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:02:11.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:10 smithi043 ceph-mon[82531]: pgmap v259: 161 pgs: 16 active+undersized+degraded, 22 peering, 3 stale+active+clean, 24 active+undersized, 96 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 48/627 objects degraded (7.656%) 2024-05-15T16:02:11.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:10 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:11.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:10 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:11.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:10 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:11.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:10 smithi043 ceph-mon[88099]: pgmap v259: 161 pgs: 16 active+undersized+degraded, 22 peering, 3 stale+active+clean, 24 active+undersized, 96 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 48/627 objects degraded (7.656%) 2024-05-15T16:02:11.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:10 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:11.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:10 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:11.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:10 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:11.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:11 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:02:11.147+0000 7fd4457d87c0 -1 Falling back to public interface 2024-05-15T16:02:11.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:10 smithi053 ceph-mon[67220]: pgmap v259: 161 pgs: 16 active+undersized+degraded, 22 peering, 3 stale+active+clean, 24 active+undersized, 96 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 48/627 objects degraded (7.656%) 2024-05-15T16:02:11.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:10 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:11.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:10 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:11.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:10 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:12.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:12 smithi043 ceph-mon[82531]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-05-15T16:02:12.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:12 smithi043 ceph-mon[88099]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-05-15T16:02:12.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:12 smithi053 ceph-mon[67220]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-05-15T16:02:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:13 smithi043 ceph-mon[82531]: pgmap v260: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T16:02:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:13 smithi043 ceph-mon[88099]: pgmap v260: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T16:02:13.366 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:13 smithi053 ceph-mon[67220]: pgmap v260: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T16:02:13.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:13 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:02:13.611+0000 7fd4457d87c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-05-15T16:02:13.919 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:13 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:02:13.646+0000 7fd4457d87c0 -1 osd.7 142 log_to_monitors true 2024-05-15T16:02:14.293 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:14 smithi053 ceph-mon[67220]: from='osd.7 [v2:172.21.15.53:6824/2465650078,v1:172.21.15.53:6825/2465650078]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T16:02:14.293 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:14 smithi053 ceph-mon[67220]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T16:02:14.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:14 smithi043 ceph-mon[88099]: from='osd.7 [v2:172.21.15.53:6824/2465650078,v1:172.21.15.53:6825/2465650078]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T16:02:14.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:14 smithi043 ceph-mon[88099]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T16:02:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:14 smithi043 ceph-mon[82531]: from='osd.7 [v2:172.21.15.53:6824/2465650078,v1:172.21.15.53:6825/2465650078]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T16:02:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:14 smithi043 ceph-mon[82531]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-15T16:02:14.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:02:14 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:02:14.290+0000 7fd432c27700 -1 osd.7 142 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-15T16:02:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:15 smithi053 ceph-mon[67220]: pgmap v261: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T16:02:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:15 smithi053 ceph-mon[67220]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-15T16:02:15.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:15 smithi053 ceph-mon[67220]: osdmap e145: 8 total, 7 up, 8 in 2024-05-15T16:02:15.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:15 smithi053 ceph-mon[67220]: from='osd.7 [v2:172.21.15.53:6824/2465650078,v1:172.21.15.53:6825/2465650078]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:02:15.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:15 smithi053 ceph-mon[67220]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:02:15.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:15 smithi053 ceph-mon[67220]: from='osd.7 ' entity='osd.7' 2024-05-15T16:02:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[88099]: pgmap v261: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T16:02:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[88099]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-15T16:02:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[88099]: osdmap e145: 8 total, 7 up, 8 in 2024-05-15T16:02:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[88099]: from='osd.7 [v2:172.21.15.53:6824/2465650078,v1:172.21.15.53:6825/2465650078]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:02:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[88099]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:02:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[88099]: from='osd.7 ' entity='osd.7' 2024-05-15T16:02:15.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[82531]: pgmap v261: 161 pgs: 27 active+undersized+degraded, 42 active+undersized, 92 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-05-15T16:02:15.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[82531]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-15T16:02:15.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[82531]: osdmap e145: 8 total, 7 up, 8 in 2024-05-15T16:02:15.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[82531]: from='osd.7 [v2:172.21.15.53:6824/2465650078,v1:172.21.15.53:6825/2465650078]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:02:15.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[82531]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-05-15T16:02:15.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:15 smithi043 ceph-mon[82531]: from='osd.7 ' entity='osd.7' 2024-05-15T16:02:16.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:02:16] "GET /metrics HTTP/1.1" 200 38285 "" "Prometheus/2.43.0" 2024-05-15T16:02:16.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:16.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:16.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[88099]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[88099]: osd.7 [v2:172.21.15.53:6824/2465650078,v1:172.21.15.53:6825/2465650078] boot 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[88099]: osdmap e146: 8 total, 8 up, 8 in 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[88099]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 27 pgs degraded (PG_DEGRADED) 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[82531]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[82531]: osd.7 [v2:172.21.15.53:6824/2465650078,v1:172.21.15.53:6825/2465650078] boot 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[82531]: osdmap e146: 8 total, 8 up, 8 in 2024-05-15T16:02:16.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:16 smithi043 ceph-mon[82531]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 27 pgs degraded (PG_DEGRADED) 2024-05-15T16:02:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:16 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:16 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:16 smithi053 ceph-mon[67220]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-15T16:02:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:16 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-15T16:02:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:16 smithi053 ceph-mon[67220]: osd.7 [v2:172.21.15.53:6824/2465650078,v1:172.21.15.53:6825/2465650078] boot 2024-05-15T16:02:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:16 smithi053 ceph-mon[67220]: osdmap e146: 8 total, 8 up, 8 in 2024-05-15T16:02:16.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:16 smithi053 ceph-mon[67220]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 27 pgs degraded (PG_DEGRADED) 2024-05-15T16:02:17.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:17 smithi043 ceph-mon[88099]: pgmap v264: 161 pgs: 4 peering, 26 active+undersized+degraded, 39 active+undersized, 92 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-05-15T16:02:17.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:17 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:17.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:17 smithi043 ceph-mon[88099]: osdmap e147: 8 total, 8 up, 8 in 2024-05-15T16:02:17.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:17 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:17.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:17 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:17.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:17 smithi043 ceph-mon[82531]: pgmap v264: 161 pgs: 4 peering, 26 active+undersized+degraded, 39 active+undersized, 92 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-05-15T16:02:17.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:17 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:17.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:17 smithi043 ceph-mon[82531]: osdmap e147: 8 total, 8 up, 8 in 2024-05-15T16:02:17.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:17 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:17.587 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:17 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:17 smithi053 ceph-mon[67220]: pgmap v264: 161 pgs: 4 peering, 26 active+undersized+degraded, 39 active+undersized, 92 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-05-15T16:02:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:17 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:17 smithi053 ceph-mon[67220]: osdmap e147: 8 total, 8 up, 8 in 2024-05-15T16:02:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:17 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:17.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:17 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:19 smithi053 ceph-mon[67220]: pgmap v266: 161 pgs: 10 peering, 18 active+undersized+degraded, 29 active+undersized, 104 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-05-15T16:02:19.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:19 smithi043 ceph-mon[88099]: pgmap v266: 161 pgs: 10 peering, 18 active+undersized+degraded, 29 active+undersized, 104 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-05-15T16:02:19.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:19 smithi043 ceph-mon[82531]: pgmap v266: 161 pgs: 10 peering, 18 active+undersized+degraded, 29 active+undersized, 104 active+clean; 585 KiB data, 252 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-05-15T16:02:20.958 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: pgmap v267: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: pgmap v268: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 847 B/s rd, 0 op/s 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: pgmap v269: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:20.959 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[82531]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/627 objects degraded (9.569%), 18 pgs degraded) 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: pgmap v267: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: pgmap v268: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 847 B/s rd, 0 op/s 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: pgmap v269: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:20.960 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:20.961 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:20.961 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:20.961 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:20.961 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:20.961 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:20 smithi043 ceph-mon[88099]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/627 objects degraded (9.569%), 18 pgs degraded) 2024-05-15T16:02:21.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: pgmap v267: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:21.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: pgmap v268: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 847 B/s rd, 0 op/s 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: pgmap v269: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:21.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:20 smithi053 ceph-mon[67220]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/627 objects degraded (9.569%), 18 pgs degraded) 2024-05-15T16:02:22.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:02:22.158+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:22.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:02:22.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:22.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:22.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:22.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:22.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:22.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:22.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:22.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:02:22.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:22.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:02:22.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:22.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:22.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:22.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:22.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:22.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:22.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:22.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:22.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:22.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:22 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:23.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:23.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:23.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:23.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all osd 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-15T16:02:23.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-15T16:02:23.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-15T16:02:23.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-15T16:02:23.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-15T16:02:23.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-15T16:02:23.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-15T16:02:23.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-15T16:02:23.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-15T16:02:23.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-15T16:02:23.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: Upgrade: Setting require_osd_release to 19 squid 2024-05-15T16:02:23.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-15T16:02:23.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:23 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-15T16:02:23.438 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:23.439 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:23.440 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:23.441 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:23.442 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:23.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-05-15T16:02:23.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:23.443 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:23.443 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:23.443 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:23.443 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:23.443 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.443 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.443 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:23.443 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:23.444 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:23.445 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:23.446 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:23.447 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:23.447 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:23.447 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:23.447 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:23.447 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.447 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:23.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all osd 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-15T16:02:23.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all osd 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-15T16:02:23.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-15T16:02:23.451 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-15T16:02:23.451 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-15T16:02:23.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-15T16:02:23.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-15T16:02:23.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-15T16:02:23.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-15T16:02:23.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-15T16:02:23.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-15T16:02:23.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-15T16:02:23.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: Upgrade: Setting require_osd_release to 19 squid 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-15T16:02:23.452 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-15T16:02:23.453 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-15T16:02:23.453 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-15T16:02:23.453 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-15T16:02:23.453 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-15T16:02:23.453 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-15T16:02:23.453 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-15T16:02:23.453 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-15T16:02:23.453 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: Upgrade: Setting require_osd_release to 19 squid 2024-05-15T16:02:23.453 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-15T16:02:23.453 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:23 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-15T16:02:24.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-15T16:02:24.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: osdmap e148: 8 total, 8 up, 8 in 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mds 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all nfs 2024-05-15T16:02:24.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all nvmeof 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: Upgrade: Finalizing container_image settings 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T16:02:24.382 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T16:02:24.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:02:24.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:02:24.383 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:02:24.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: Upgrade: Complete! 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:24.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:24.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-05-15T16:02:24.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T16:02:24.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:24.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:24.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:24 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: osdmap e148: 8 total, 8 up, 8 in 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mds 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.442 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all nfs 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all nvmeof 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.443 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: Upgrade: Finalizing container_image settings 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:02:24.444 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:02:24.445 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:02:24.446 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:02:24.446 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T16:02:24.446 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-15T16:02:24.446 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-15T16:02:24.446 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: osdmap e148: 8 total, 8 up, 8 in 2024-05-15T16:02:24.446 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.446 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mds 2024-05-15T16:02:24.446 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.446 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.446 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all nfs 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all nvmeof 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.447 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:24.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: Upgrade: Finalizing container_image settings 2024-05-15T16:02:24.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:02:24.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:02:24.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T16:02:24.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.448 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.449 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: Upgrade: Complete! 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:24.450 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:24.451 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:24.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:02:24.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:02:24.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T16:02:24.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:02:24.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:02:24.451 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:02:24.452 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.453 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: Upgrade: Complete! 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-05-15T16:02:24.454 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:24.455 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:24.455 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:24.455 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:24 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:25.417 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:25 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:25.418 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:25 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:25.418 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:25 smithi043 ceph-mon[82531]: from='client.44607 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:25.418 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:25 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:25.418 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:25 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:25.418 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:25 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:25.418 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:25 smithi043 ceph-mon[88099]: from='client.44607 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:25.418 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:25 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:25.420 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T16:02:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:25 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:25 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:25 smithi053 ceph-mon[67220]: from='client.44607 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:25.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:25 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:26.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:02:25.978+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:26.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:02:26.086 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:26.087 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:26.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:26.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:26.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:26.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:26.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:26.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:26.088 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:26.089 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:26.090 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:26.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:26.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:26.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:26.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:26.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:26.091 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:25 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:26.468 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[88099]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-05-15T16:02:26.468 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:26.468 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:26.468 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:26.468 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:26.468 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[82531]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:26 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:26.469 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:02:26] "GET /metrics HTTP/1.1" 200 38368 "" "Prometheus/2.43.0" 2024-05-15T16:02:26.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:26 smithi053 ceph-mon[67220]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 971 B/s rd, 0 op/s 2024-05-15T16:02:26.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:26 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:26.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:26 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:26.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:26 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:26.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:26 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:26.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:26 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:26.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:26 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:26.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:26 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:27.390 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:27.390 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:27.390 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:27.390 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:27.390 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:27.390 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:27.390 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:27.391 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:27.392 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:27.393 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:27.394 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:27.394 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:27.394 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:27.394 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:27.394 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:27.394 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:27.394 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:27.394 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:27.394 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:27.394 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:27.395 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:02:27.396 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:27.397 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[88099]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-05-15T16:02:27.398 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[88099]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[82531]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[82531]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:27.399 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:27 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:27 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:27.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:27 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:27.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:27.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:27.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:27.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:27.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:27.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:27 smithi053 ceph-mon[67220]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-05-15T16:02:27.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:27 smithi053 ceph-mon[67220]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:02:27.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:27 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:27.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:27 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (9m) 2m ago 31m 26.6M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (9m) 12s ago 31m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 2m ago 30m 65.1M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (10m) 12s ago 36m 455M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (20m) 2m ago 37m 512M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (7m) 2m ago 37m 52.0M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (7m) 12s ago 36m 53.4M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (6m) 2m ago 36m 42.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (10m) 2m ago 32m 15.1M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T16:02:27.865 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (10m) 12s ago 32m 16.2M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (4m) 2m ago 35m 45.6M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (4m) 2m ago 35m 44.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (3m) 2m ago 35m 49.7M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (2m) 2m ago 34m 12.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 258344f90360 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (2m) 12s ago 34m 55.7M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 b342d0fab575 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (91s) 12s ago 33m 53.7M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 f709584ed028 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (54s) 12s ago 33m 45.0M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 d5353ef72311 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (17s) 12s ago 33m 12.1M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 784f18f73ebd 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (9m) 12s ago 32m 77.6M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (30m) 2m ago 30m 78.6M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T16:02:27.866 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (30m) 12s ago 30m 78.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T16:02:28.664 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-05-15T16:02:29.468 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:29 smithi043 ceph-mon[82531]: from='client.44613 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:29.469 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:29 smithi043 ceph-mon[82531]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T16:02:29.469 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:29 smithi043 ceph-mon[88099]: from='client.44613 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:29.469 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:29 smithi043 ceph-mon[88099]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T16:02:29.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:29 smithi053 ceph-mon[67220]: from='client.44613 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:29.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:29 smithi053 ceph-mon[67220]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-05-15T16:02:30.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:30 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:30.978 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:30 smithi043 ceph-mon[82531]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T16:02:30.978 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:30 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:30.978 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:30 smithi043 ceph-mon[88099]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T16:02:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:30 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:30 smithi053 ceph-mon[67220]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-05-15T16:02:31.336 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:02:31.052+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:31.337 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:31.338 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:31.339 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:31.340 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:31.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:31.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:31.341 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:31 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:31.518 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T16:02:32.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:31 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:32.086 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:32.086 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:32.086 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:32.086 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:32.086 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.086 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.087 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:32.088 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:32.089 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:32.090 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:31 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:32.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:32.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:32.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:32.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:32.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:32.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:32.090 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:32.091 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:32.092 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:32.093 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:31 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1601023466' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.094 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.095 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:31 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1601023466' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:31 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:32.130 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:32.131 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:02:32.132 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:32.133 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:32.134 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:31 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1601023466' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:32.380 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-05-15T16:02:33.027 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:32 smithi043 ceph-mon[82531]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:33.028 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:32 smithi043 ceph-mon[88099]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:33.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:32 smithi053 ceph-mon[67220]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:34.517 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)" 2024-05-15T16:02:35.163 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:35 smithi043 ceph-mon[82531]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:35.164 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:35 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3278581734' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:35.164 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:35 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:35.164 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:35.164 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:35 smithi043 ceph-mon[88099]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:35.164 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:35 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3278581734' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:35.164 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:35 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:35.164 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:35.190 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade status' 2024-05-15T16:02:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:35 smithi053 ceph-mon[67220]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:35 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3278581734' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:35 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:36.487 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:02:36] "GET /metrics HTTP/1.1" 200 38365 "" "Prometheus/2.43.0" 2024-05-15T16:02:37.261 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:02:37.261 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": null, 2024-05-15T16:02:37.261 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": false, 2024-05-15T16:02:37.261 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "", 2024-05-15T16:02:37.261 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T16:02:37.262 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": null, 2024-05-15T16:02:37.262 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "", 2024-05-15T16:02:37.262 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T16:02:37.262 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:02:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:37 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:37 smithi043 ceph-mon[82531]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:02:37.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:37 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:37.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:37 smithi043 ceph-mon[88099]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:02:37.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:37 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:37.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:37 smithi053 ceph-mon[67220]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:02:37.855 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T16:02:38.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:38 smithi043 ceph-mon[82531]: from='client.44631 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:38.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:38 smithi043 ceph-mon[88099]: from='client.44631 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:38 smithi053 ceph-mon[67220]: from='client.44631 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:39.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:39 smithi043 ceph-mon[82531]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:39.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:39 smithi043 ceph-mon[88099]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:39.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:39 smithi053 ceph-mon[67220]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:40.165 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T16:02:40.165 INFO:teuthology.orchestra.run.smithi043.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T16:02:40.165 INFO:teuthology.orchestra.run.smithi043.stdout: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T16:02:40.978 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-05-15T16:02:41.300 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:41 smithi043 ceph-mon[82531]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:41.300 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:41 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/799697319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:02:41.300 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:41 smithi043 ceph-mon[88099]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:41.301 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:41 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/799697319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:02:41.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:41 smithi053 ceph-mon[67220]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:41.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:41 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/799697319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:02:43.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:43 smithi043 ceph-mon[82531]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:43.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:43 smithi043 ceph-mon[88099]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:43 smithi053 ceph-mon[67220]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:44.308 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:44 smithi043 ceph-mon[82531]: from='client.44643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:44.308 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:44 smithi043 ceph-mon[88099]: from='client.44643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:44.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:44 smithi053 ceph-mon[67220]: from='client.44643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:45.306 INFO:teuthology.orchestra.run.smithi043.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:02:45.322 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:45 smithi043 ceph-mon[82531]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:45.323 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:45 smithi043 ceph-mon[88099]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:45.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:45 smithi053 ceph-mon[67220]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:46.068 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-05-15T16:02:46.579 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[82531]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:02:46.579 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:46.579 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:46.579 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:46.580 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:46.580 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:46.580 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[88099]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:02:46.580 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:46.580 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:46.580 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:46.580 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:46.580 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:46 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:46.580 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:02:46] "GET /metrics HTTP/1.1" 200 38365 "" "Prometheus/2.43.0" 2024-05-15T16:02:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:46 smithi053 ceph-mon[67220]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:02:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:46 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:46 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:46 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:02:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:46 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:47.482 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:47.482 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[82531]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:47.482 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[82531]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-05-15T16:02:47.482 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[82531]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 617 B/s rd, 0 op/s 2024-05-15T16:02:47.482 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:47.482 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:47.482 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:47.482 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:47.482 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[88099]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:47.483 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[88099]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-05-15T16:02:47.483 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[88099]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 617 B/s rd, 0 op/s 2024-05-15T16:02:47.483 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:47.483 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:47.483 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:47 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:47 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:47 smithi053 ceph-mon[67220]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:02:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:47 smithi053 ceph-mon[67220]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-05-15T16:02:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:47 smithi053 ceph-mon[67220]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 617 B/s rd, 0 op/s 2024-05-15T16:02:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:47 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:47 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:02:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:47 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:48.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:48 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:48.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:48 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:48.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:48 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:48.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:48 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:48.452 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T16:02:48.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:48 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:02:48.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:48 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:02:49.140 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T16:02:49.141 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (9m) 2m ago 31m 26.6M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T16:02:49.141 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (9m) 33s ago 31m 82.4M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T16:02:49.141 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 2m ago 31m 65.1M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T16:02:49.141 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (11m) 33s ago 36m 455M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T16:02:49.141 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (21m) 2m ago 38m 512M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T16:02:49.141 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (7m) 2m ago 38m 52.0M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T16:02:49.141 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (8m) 33s ago 36m 53.4M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T16:02:49.141 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (6m) 2m ago 36m 42.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T16:02:49.141 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (10m) 2m ago 32m 15.1M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T16:02:49.141 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (10m) 33s ago 32m 16.2M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (5m) 2m ago 36m 45.6M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (4m) 2m ago 35m 44.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (3m) 2m ago 35m 49.7M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (2m) 2m ago 34m 12.3M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 258344f90360 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (2m) 33s ago 34m 55.7M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 b342d0fab575 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (112s) 33s ago 34m 53.7M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 f709584ed028 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (75s) 33s ago 33m 45.0M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 d5353ef72311 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (38s) 33s ago 33m 12.1M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 784f18f73ebd 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (10m) 33s ago 32m 77.6M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (31m) 2m ago 31m 78.6M - 17.2.0 e1d6a67b021e c9796ddfa4f9 2024-05-15T16:02:49.142 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (31m) 33s ago 31m 78.0M - 17.2.0 e1d6a67b021e 564a76474d0a 2024-05-15T16:02:49.420 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:49 smithi043 ceph-mon[88099]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:49.420 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:49 smithi043 ceph-mon[82531]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:49.567 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:02:49.567 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T16:02:49.567 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T16:02:49.567 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:02:49.567 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 8 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 13 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T16:02:49.568 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:02:49.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:49 smithi053 ceph-mon[67220]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:49.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:02:49.544+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:49.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:49.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:49.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:49.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:02:49.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:49.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:49 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:49.930 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:02:49.930 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", 2024-05-15T16:02:49.930 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T16:02:49.930 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-05-15T16:02:49.931 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T16:02:49.931 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "", 2024-05-15T16:02:49.931 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 image", 2024-05-15T16:02:49.931 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T16:02:49.931 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:02:50.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:50 smithi043 ceph-mon[82531]: from='client.44649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:50.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:50 smithi043 ceph-mon[82531]: from='client.44655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:50.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:50 smithi043 ceph-mon[82531]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:50.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:50 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1100164932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:50.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:50 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:50.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:50 smithi043 ceph-mon[88099]: from='client.44649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:50.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:50 smithi043 ceph-mon[88099]: from='client.44655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:50.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:50 smithi043 ceph-mon[88099]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:50.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:50 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1100164932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:50.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:50 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:50 smithi053 ceph-mon[67220]: from='client.44649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:50 smithi053 ceph-mon[67220]: from='client.44655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:50 smithi053 ceph-mon[67220]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:50 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1100164932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:50 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:02:51.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:51 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:51.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:51.586 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:51.587 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:51.588 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.589 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:51.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:51.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:51.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:51.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:51.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:51.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:51.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:51.590 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:51.590 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:51 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:51.590 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:51.591 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:51.592 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:02:51.593 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:51.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:51.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:51.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:02:51.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:51.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:51.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:51.594 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:51.594 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:51.594 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:51.594 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:51 smithi043 ceph-mon[82531]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:51 smithi043 ceph-mon[82531]: from='client.44673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:51 smithi043 ceph-mon[82531]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 617 B/s rd, 0 op/s 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:51 smithi043 ceph-mon[88099]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:51 smithi043 ceph-mon[88099]: from='client.44673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:51.595 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:51 smithi043 ceph-mon[88099]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 617 B/s rd, 0 op/s 2024-05-15T16:02:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:51 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:51.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:51.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:02:51.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:02:51.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:51 smithi053 ceph-mon[67220]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:02:51.634 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:51 smithi053 ceph-mon[67220]: from='client.44673 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:02:51.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:51 smithi053 ceph-mon[67220]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 617 B/s rd, 0 op/s 2024-05-15T16:02:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T16:02:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T16:02:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mgr 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mon 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all crash 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all osd 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mds 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[82531]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mgr 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mon 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all crash 2024-05-15T16:02:53.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all osd 2024-05-15T16:02:53.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mds 2024-05-15T16:02:53.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.089 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:52 smithi043 ceph-mon[88099]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:53.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T16:02:53.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T16:02:53.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:02:53.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mgr 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mon 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all crash 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all osd 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mds 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:53.131 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:52 smithi053 ceph-mon[67220]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[88099]: Upgrade: Updating rgw.foo.smithi043.fiueec (1/2) 2024-05-15T16:02:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:02:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:02:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[88099]: Deploying daemon rgw.foo.smithi043.fiueec on smithi043 2024-05-15T16:02:54.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[88099]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:54.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[82531]: Upgrade: Updating rgw.foo.smithi043.fiueec (1/2) 2024-05-15T16:02:54.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:54.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:02:54.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:02:54.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:54.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[82531]: Deploying daemon rgw.foo.smithi043.fiueec on smithi043 2024-05-15T16:02:54.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:54 smithi043 ceph-mon[82531]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:54 smithi053 ceph-mon[67220]: Upgrade: Updating rgw.foo.smithi043.fiueec (1/2) 2024-05-15T16:02:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:54 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:54 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:02:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:54 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi043.fiueec", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:02:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:54 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:02:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:54 smithi053 ceph-mon[67220]: Deploying daemon rgw.foo.smithi043.fiueec on smithi043 2024-05-15T16:02:54.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:54 smithi053 ceph-mon[67220]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:02:56.500 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:02:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:02:56] "GET /metrics HTTP/1.1" 200 38366 "" "Prometheus/2.43.0" 2024-05-15T16:02:57.541 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:57 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:57.541 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:57 smithi043 ceph-mon[88099]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:02:57.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:57 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:57.541 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:57 smithi043 ceph-mon[82531]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:02:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:57 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:02:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:57 smithi053 ceph-mon[67220]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:02:59.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:58 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:59.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:58 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:59.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:02:58 smithi043 ceph-mon[88099]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 0 B/s wr, 5 op/s 2024-05-15T16:02:59.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:58 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:59.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:58 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:59.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:02:58 smithi043 ceph-mon[82531]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 0 B/s wr, 5 op/s 2024-05-15T16:02:59.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:58 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:59.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:58 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:02:59.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:02:58 smithi053 ceph-mon[67220]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 0 B/s wr, 5 op/s 2024-05-15T16:03:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[82531]: Upgrade: Updating rgw.foo.smithi053.vjmwpu (2/2) 2024-05-15T16:03:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:03:00.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:03:00.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:00.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[82531]: Deploying daemon rgw.foo.smithi053.vjmwpu on smithi053 2024-05-15T16:03:00.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[82531]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 85 B/s wr, 50 op/s 2024-05-15T16:03:00.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[88099]: Upgrade: Updating rgw.foo.smithi053.vjmwpu (2/2) 2024-05-15T16:03:00.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:00.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:03:00.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:03:00.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:00.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[88099]: Deploying daemon rgw.foo.smithi053.vjmwpu on smithi053 2024-05-15T16:03:00.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:00 smithi043 ceph-mon[88099]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 85 B/s wr, 50 op/s 2024-05-15T16:03:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:00 smithi053 ceph-mon[67220]: Upgrade: Updating rgw.foo.smithi053.vjmwpu (2/2) 2024-05-15T16:03:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:00 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:00 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:03:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:00 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi053.vjmwpu", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-15T16:03:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:00 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:00 smithi053 ceph-mon[67220]: Deploying daemon rgw.foo.smithi053.vjmwpu on smithi053 2024-05-15T16:03:00.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:00 smithi053 ceph-mon[67220]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 85 B/s wr, 50 op/s 2024-05-15T16:03:03.615 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:03 smithi053 ceph-mon[67220]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 85 B/s wr, 77 op/s 2024-05-15T16:03:03.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:03 smithi043 ceph-mon[82531]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 85 B/s wr, 77 op/s 2024-05-15T16:03:03.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:03 smithi043 ceph-mon[88099]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 85 B/s wr, 77 op/s 2024-05-15T16:03:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:04 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:04 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:04 smithi043 ceph-mon[88099]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 170 B/s wr, 122 op/s 2024-05-15T16:03:04.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:04 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:04 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:04.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:04.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:04 smithi043 ceph-mon[82531]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 170 B/s wr, 122 op/s 2024-05-15T16:03:04.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:04 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:04 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:04 smithi053 ceph-mon[67220]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 170 B/s wr, 122 op/s 2024-05-15T16:03:04.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:06.564 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:03:06] "GET /metrics HTTP/1.1" 200 38366 "" "Prometheus/2.43.0" 2024-05-15T16:03:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:07 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:07.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:07 smithi053 ceph-mon[67220]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 170 B/s wr, 135 op/s 2024-05-15T16:03:07.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:07 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:07.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:07 smithi043 ceph-mon[88099]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 170 B/s wr, 135 op/s 2024-05-15T16:03:07.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:07 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:07.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:07 smithi043 ceph-mon[82531]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 170 B/s wr, 135 op/s 2024-05-15T16:03:08.835 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:08 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:08 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:08 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:08 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:08 smithi043 ceph-mon[82531]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 170 B/s wr, 162 op/s 2024-05-15T16:03:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:08 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:08 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:08 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:08 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.836 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:08 smithi043 ceph-mon[88099]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 170 B/s wr, 162 op/s 2024-05-15T16:03:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:08 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:08 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:08 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:08 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:08.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:08 smithi053 ceph-mon[67220]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 170 B/s wr, 162 op/s 2024-05-15T16:03:09.634 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:09 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.634 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:09 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.634 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:09 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.635 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:09 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.635 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:09 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.635 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:09 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.635 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:09 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.635 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:09 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:09 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:09 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:09 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:09.880 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:09 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:10.674 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:10 smithi043 ceph-mon[82531]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 170 B/s wr, 163 op/s 2024-05-15T16:03:10.674 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:10 smithi043 ceph-mon[88099]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 170 B/s wr, 163 op/s 2024-05-15T16:03:10.879 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:10 smithi053 ceph-mon[67220]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 170 B/s wr, 163 op/s 2024-05-15T16:03:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 85 B/s wr, 118 op/s 2024-05-15T16:03:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:13.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 85 B/s wr, 118 op/s 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:13.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:13.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:13.338 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:13 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:13.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 85 B/s wr, 118 op/s 2024-05-15T16:03:13.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:13.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:13.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:13.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:13.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:13.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:13 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:14 smithi043 ceph-mon[82531]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 101 B/s wr, 109 op/s 2024-05-15T16:03:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:14 smithi043 ceph-mon[82531]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 67 op/s 2024-05-15T16:03:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:14 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:14.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:14 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:14.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:14 smithi043 ceph-mon[88099]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 101 B/s wr, 109 op/s 2024-05-15T16:03:14.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:14 smithi043 ceph-mon[88099]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 67 op/s 2024-05-15T16:03:14.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:14 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:14.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:14 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:14 smithi053 ceph-mon[67220]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 101 B/s wr, 109 op/s 2024-05-15T16:03:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:14 smithi053 ceph-mon[67220]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 67 op/s 2024-05-15T16:03:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:14 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:14.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:14 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:03:14.334+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:03:14.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:14.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:03:14.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:03:14.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:14.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:03:14.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:14.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:14.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:14.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:14.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:14.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:14.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:14.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:14.842 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:14 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.337 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:15.338 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:15.339 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.340 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:15.341 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:15.342 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:15.343 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.344 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all rgw 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi043.fiueec"}]: dispatch 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi043.fiueec"}]: dispatch 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi043.fiueec"}]': finished 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi053.vjmwpu"}]: dispatch 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi053.vjmwpu"}]: dispatch 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi053.vjmwpu"}]': finished 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.345 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.346 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.346 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:15.346 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:15.346 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:15.346 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.346 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.346 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all rgw 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi043.fiueec"}]: dispatch 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi043.fiueec"}]: dispatch 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi043.fiueec"}]': finished 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi053.vjmwpu"}]: dispatch 2024-05-15T16:03:15.347 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi053.vjmwpu"}]: dispatch 2024-05-15T16:03:15.348 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi053.vjmwpu"}]': finished 2024-05-15T16:03:15.348 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.348 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.348 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.348 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.348 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.348 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.348 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.349 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:03:15.350 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.351 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T16:03:15.352 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:15.353 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:15.353 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:15.353 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.353 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.353 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:15.353 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:15.353 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:15.353 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T16:03:15.354 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:03:15.355 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T16:03:15.356 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.357 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:15.358 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:15 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:15.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:15.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:15.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.384 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all rgw 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi043.fiueec"}]: dispatch 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi043.fiueec"}]: dispatch 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi043.fiueec"}]': finished 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi053.vjmwpu"}]: dispatch 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi053.vjmwpu"}]: dispatch 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi053.vjmwpu"}]': finished 2024-05-15T16:03:15.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.386 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-15T16:03:15.387 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:03:15.388 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-15T16:03:15.388 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-15T16:03:15.388 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:03:15.388 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-15T16:03:15.388 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-15T16:03:15.388 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:03:15.388 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:03:15.389 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.390 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:15.391 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:15 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T16:03:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T16:03:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 49 op/s 2024-05-15T16:03:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all nfs 2024-05-15T16:03:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all nvmeof 2024-05-15T16:03:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: Upgrade: Finalizing container_image settings 2024-05-15T16:03:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: Upgrade: Complete! 2024-05-15T16:03:16.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T16:03:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: Checking dashboard <-> RGW credentials 2024-05-15T16:03:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:16.087 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:16.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T16:03:16.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T16:03:16.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 49 op/s 2024-05-15T16:03:16.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all nfs 2024-05-15T16:03:16.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all nvmeof 2024-05-15T16:03:16.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: Upgrade: Finalizing container_image settings 2024-05-15T16:03:16.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: Upgrade: Complete! 2024-05-15T16:03:16.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T16:03:16.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:16.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: Checking dashboard <-> RGW credentials 2024-05-15T16:03:16.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:16.088 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:16 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:16.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T16:03:16.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T16:03:16.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 49 op/s 2024-05-15T16:03:16.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all nfs 2024-05-15T16:03:16.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all nvmeof 2024-05-15T16:03:16.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: Upgrade: Finalizing container_image settings 2024-05-15T16:03:16.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: Upgrade: Complete! 2024-05-15T16:03:16.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 0 B/s wr, 12 op/s 2024-05-15T16:03:16.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:16.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: Checking dashboard <-> RGW credentials 2024-05-15T16:03:16.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:16.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:16 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:16.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:03:16] "GET /metrics HTTP/1.1" 200 38366 "" "Prometheus/2.43.0" 2024-05-15T16:03:16.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:03:16.568+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:16.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:03:16.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:16.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:16.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:16.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:16.837 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:16.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:16.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:16.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:16.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:16.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:16.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:16.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:16.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:16.838 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:03:16.839 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:16.840 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:16.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:16.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:16.841 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:17.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:17.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:17.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:17.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:17.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:17.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:17.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:17.337 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:17 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:17 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:17 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:17 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:17 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:17 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:17 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:17 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:17 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:18.313 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[82531]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:18.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:18.315 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:18.316 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:18.316 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:18.316 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:03:18.316 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:18.316 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:03:18.316 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:18.316 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:03:18.316 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:18.316 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:18.317 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:18.318 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:18.318 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:18.318 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:18.318 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:18.318 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:18.318 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:18.318 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:18.318 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[88099]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:18.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:18.320 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:18.321 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:18.322 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:18.323 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:18.323 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:18.323 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:18.323 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:18.323 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:18.323 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:18.323 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.323 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.323 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[82531]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[82531]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[88099]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[88099]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:18.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:18 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:18.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:18 smithi053 ceph-mon[67220]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-05-15T16:03:18.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:18 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:18.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:18.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:18.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:18.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:18.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.385 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:18 smithi053 ceph-mon[67220]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-05-15T16:03:18.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:18 smithi053 ceph-mon[67220]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-05-15T16:03:18.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:18 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:18.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:18 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:18.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:03:18.436+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:18.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:03:18.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:18.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:18.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:18.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.586 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:18.587 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:18.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:18.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:18.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:18.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:18.588 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:18.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:18.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:18.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:18.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:03:18.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:18.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:03:18.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:18.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:03:18.589 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:03:18.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:18.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:03:18.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:18.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:18.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:18.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:18.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:18.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:18.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:18.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:18.591 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:18 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.311 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:20 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:20.311 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:20.311 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:20.311 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:20.311 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:20.311 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.311 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.311 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:20.312 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:20.313 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:20.314 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:20 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:20.315 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:20.316 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:20.317 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:20.318 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:20 smithi043 ceph-mon[88099]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 3 op/s 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:20 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:20 smithi043 ceph-mon[82531]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 3 op/s 2024-05-15T16:03:20.319 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:20 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:20.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:20 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:20.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:20.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:20.380 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:20.381 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:03:20.382 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.383 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:20.384 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:20.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:20 smithi053 ceph-mon[67220]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 3 op/s 2024-05-15T16:03:20.385 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:20 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:20.927 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T16:03:21.246 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:21 smithi043 ceph-mon[82531]: from='client.54697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:21.246 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:21 smithi043 ceph-mon[88099]: from='client.54697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:21.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:21 smithi053 ceph-mon[67220]: from='client.54697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:22.307 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:22 smithi043 ceph-mon[82531]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 4 op/s 2024-05-15T16:03:22.308 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:22 smithi043 ceph-mon[88099]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 4 op/s 2024-05-15T16:03:22.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:22 smithi053 ceph-mon[67220]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 4 op/s 2024-05-15T16:03:22.896 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (10m) 15s ago 32m 25.7M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (9m) 15s ago 32m 83.0M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 15s ago 31m 65.3M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (11m) 15s ago 37m 455M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (21m) 15s ago 38m 519M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (7m) 15s ago 38m 60.7M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (8m) 15s ago 37m 56.5M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (7m) 15s ago 37m 50.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (11m) 15s ago 33m 15.5M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (11m) 15s ago 33m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (5m) 15s ago 36m 52.8M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T16:03:22.897 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (5m) 15s ago 36m 53.9M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T16:03:22.898 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (4m) 15s ago 35m 56.1M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T16:03:22.898 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (3m) 15s ago 35m 59.5M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 258344f90360 2024-05-15T16:03:22.898 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (3m) 15s ago 35m 59.3M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 b342d0fab575 2024-05-15T16:03:22.898 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (2m) 15s ago 34m 57.4M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 f709584ed028 2024-05-15T16:03:22.898 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (109s) 15s ago 34m 47.8M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 d5353ef72311 2024-05-15T16:03:22.898 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (72s) 15s ago 34m 55.0M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 784f18f73ebd 2024-05-15T16:03:22.898 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (10m) 15s ago 33m 79.7M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T16:03:22.898 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (25s) 15s ago 31m 74.3M - 19.0.0-2576-gde5210d5 48bc18eb22d8 77771618aae7 2024-05-15T16:03:22.898 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (19s) 15s ago 31m 74.9M - 19.0.0-2576-gde5210d5 48bc18eb22d8 64952a4875cd 2024-05-15T16:03:23.630 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-05-15T16:03:24.260 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:24 smithi043 ceph-mon[82531]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 4 op/s 2024-05-15T16:03:24.260 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:24 smithi043 ceph-mon[82531]: from='client.44760 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:24.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:24 smithi043 ceph-mon[88099]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 4 op/s 2024-05-15T16:03:24.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:24 smithi043 ceph-mon[88099]: from='client.44760 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:24.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:24 smithi053 ceph-mon[67220]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 4 op/s 2024-05-15T16:03:24.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:24 smithi053 ceph-mon[67220]: from='client.44760 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:25.709 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T16:03:26.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:26 smithi043 ceph-mon[88099]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-05-15T16:03:26.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:26 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3271699791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:26 smithi043 ceph-mon[82531]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-05-15T16:03:26.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:26 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3271699791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:26.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:26 smithi053 ceph-mon[67220]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-05-15T16:03:26.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:26 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3271699791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:26.524 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-05-15T16:03:26.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:03:26] "GET /metrics HTTP/1.1" 200 38398 "" "Prometheus/2.43.0" 2024-05-15T16:03:27.174 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:27 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:27.175 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:27 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:27.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:27 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:27.880 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:03:27 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=cleanup t=2024-05-15T16:03:27.578744768Z level=info msg="Completed cleanup jobs" duration=1.302646ms 2024-05-15T16:03:28.246 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:28 smithi043 ceph-mon[82531]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-05-15T16:03:28.246 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:28 smithi043 ceph-mon[88099]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-05-15T16:03:28.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:28 smithi053 ceph-mon[67220]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-05-15T16:03:28.809 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)" 2024-05-15T16:03:29.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:29 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3200440237' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:29.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:29 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3200440237' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:29 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3200440237' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:29.497 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade status' 2024-05-15T16:03:30.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:30 smithi043 ceph-mon[82531]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-05-15T16:03:30.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:30 smithi043 ceph-mon[88099]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-05-15T16:03:30.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:30 smithi053 ceph-mon[67220]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-05-15T16:03:31.546 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:03:31.546 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": null, 2024-05-15T16:03:31.546 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": false, 2024-05-15T16:03:31.546 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "", 2024-05-15T16:03:31.546 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [], 2024-05-15T16:03:31.546 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": null, 2024-05-15T16:03:31.547 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "", 2024-05-15T16:03:31.547 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T16:03:31.547 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:03:32.202 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:32 smithi043 ceph-mon[82531]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-05-15T16:03:32.202 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:32 smithi043 ceph-mon[88099]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-05-15T16:03:32.228 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T16:03:32.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:32 smithi053 ceph-mon[67220]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-05-15T16:03:33.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:33 smithi043 ceph-mon[82531]: from='client.54721 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:33.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:33 smithi043 ceph-mon[88099]: from='client.54721 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:33.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:33 smithi053 ceph-mon[67220]: from='client.54721 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:34.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:34 smithi043 ceph-mon[82531]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:34.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:34 smithi043 ceph-mon[88099]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:34.364 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T16:03:34.365 INFO:teuthology.orchestra.run.smithi043.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T16:03:34.365 INFO:teuthology.orchestra.run.smithi043.stdout: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T16:03:34.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:34 smithi053 ceph-mon[67220]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:34.957 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-15T16:03:35.274 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:35 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1341227899' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:03:35.274 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:35 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:35.274 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:35 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1341227899' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:03:35.275 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:35 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:35 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1341227899' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:03:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:35 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:36.309 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:36 smithi043 ceph-mon[82531]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:36.309 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:36 smithi043 ceph-mon[88099]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:36.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:36 smithi053 ceph-mon[67220]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:36.585 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:03:36] "GET /metrics HTTP/1.1" 200 38394 "" "Prometheus/2.43.0" 2024-05-15T16:03:37.056 INFO:teuthology.orchestra.run.smithi043.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:03:37.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:37 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:37 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:37.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:37 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:37.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:37 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:37.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:37 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:37.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:37 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:37.907 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-15T16:03:37.921 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi043.front.sepia.ceph.com 2024-05-15T16:03:37.922 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-05-15T16:03:38.135 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:38.135 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: from='client.44790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:38.135 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:03:38.135 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:38.135 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:38.135 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:38.135 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-05-15T16:03:38.135 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: from='client.44790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:38.136 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-05-15T16:03:38.137 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:38.137 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:38.137 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:38.137 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:38.137 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:38.137 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:38.137 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:38 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: from='client.44790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:03:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-05-15T16:03:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:38.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:38.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:38.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:38.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:38.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:38.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:38 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:40.165 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:03:39.914+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:40.165 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:03:40.165 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:40.166 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:40.167 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return future.result(timeout) 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return self.__get_result() 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: raise self._exception 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: use_current_daemon_image=reconfig, 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Creating ceph-iscsi config... 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:40.168 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: Traceback (most recent call last): 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: return _run_code(code, main_globals, None, 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: exec(code, run_globals) 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:40.169 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:40.170 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:40.170 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:40.170 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:40.170 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:39 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:40.417 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:40 smithi043 ceph-mon[82531]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:40.417 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:40 smithi043 ceph-mon[88099]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:40.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:40 smithi053 ceph-mon[67220]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:41.214 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T16:03:41.253 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:41 smithi043 ceph-mon[82531]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:41.254 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:41.255 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: return future.result(timeout) 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: return self.__get_result() 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: raise self._exception 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:41.256 INFO:journalctl@ceph.mon.a.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.257 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:41 smithi043 ceph-mon[82531]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:03:41.258 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:41 smithi043 ceph-mon[88099]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:41.258 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:41.258 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:41.258 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:41.258 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:41.258 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.258 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.259 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: return future.result(timeout) 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: return self.__get_result() 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: raise self._exception 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:41.260 INFO:journalctl@ceph.mon.c.smithi043.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: Traceback (most recent call last): 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:03:41.261 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:41.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:41.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:41.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:41.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:41.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:41.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:41.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.262 INFO:journalctl@ceph.mon.c.smithi043.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.262 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:41 smithi043 ceph-mon[88099]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:03:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:41 smithi053 ceph-mon[67220]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:41.630 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:41.631 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.event_loop.get_result(coro, timeout) 2024-05-15T16:03:41.632 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return future.result(timeout) 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: return self.__get_result() 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: raise self._exception 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: use_current_daemon_image=reconfig, 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Creating ceph-iscsi config... 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:03:41.633 INFO:journalctl@ceph.mon.b.smithi053.stdout: Non-zero exit code 1 from systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: Traceback (most recent call last): 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: exec(code, run_globals) 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1123, in deploy_daemon 2024-05-15T16:03:41.634 INFO:journalctl@ceph.mon.b.smithi053.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:03:41.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: RuntimeError: Failed command: systemctl restart ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:03:41.635 INFO:journalctl@ceph.mon.b.smithi053.stdout: See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:03:41.635 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:41 smithi053 ceph-mon[67220]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:de5210d58560a4684fdb5b238ef3e6e0285a1de8 2024-05-15T16:03:41.912 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (10m) 34s ago 32m 25.7M - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (10m) 34s ago 32m 83.0M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 unknown 34s ago 31m 65.3M - 3.5 e1d6a67b021e 8f7ce2426364 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (11m) 34s ago 37m 455M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (22m) 34s ago 38m 519M - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (8m) 34s ago 39m 60.7M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (9m) 34s ago 37m 56.5M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (7m) 34s ago 37m 50.2M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (11m) 34s ago 33m 15.5M - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (11m) 34s ago 33m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (6m) 34s ago 37m 52.8M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (5m) 34s ago 36m 53.9M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T16:03:41.913 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (4m) 34s ago 36m 56.1M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T16:03:41.914 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (3m) 34s ago 35m 59.5M 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 258344f90360 2024-05-15T16:03:41.914 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (3m) 34s ago 35m 59.3M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 b342d0fab575 2024-05-15T16:03:41.914 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (2m) 34s ago 35m 57.4M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 f709584ed028 2024-05-15T16:03:41.914 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (2m) 34s ago 34m 47.8M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 d5353ef72311 2024-05-15T16:03:41.914 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (91s) 34s ago 34m 55.0M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 784f18f73ebd 2024-05-15T16:03:41.914 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (11m) 34s ago 33m 79.7M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T16:03:41.914 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (44s) 34s ago 32m 74.3M - 19.0.0-2576-gde5210d5 48bc18eb22d8 77771618aae7 2024-05-15T16:03:41.914 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (38s) 34s ago 32m 74.9M - 19.0.0-2576-gde5210d5 48bc18eb22d8 64952a4875cd 2024-05-15T16:03:42.336 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:03:42.336 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T16:03:42.336 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T16:03:42.336 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:03:42.336 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T16:03:42.336 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:03:42.336 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:03:42.336 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T16:03:42.336 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 8 2024-05-15T16:03:42.337 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:03:42.337 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T16:03:42.337 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:03:42.337 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:03:42.337 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T16:03:42.337 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 15 2024-05-15T16:03:42.337 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T16:03:42.337 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:03:42.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:42 smithi043 ceph-mon[82531]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-05-15T16:03:42.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:42 smithi043 ceph-mon[82531]: from='client.44796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:42.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:42 smithi043 ceph-mon[88099]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-05-15T16:03:42.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:42 smithi043 ceph-mon[88099]: from='client.44796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:42 smithi053 ceph-mon[67220]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-05-15T16:03:42.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:42 smithi053 ceph-mon[67220]: from='client.44796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:42.694 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:03:42.694 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T16:03:42.694 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T16:03:42.694 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-15T16:03:42.695 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [ 2024-05-15T16:03:42.695 INFO:teuthology.orchestra.run.smithi043.stdout: "mon", 2024-05-15T16:03:42.695 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr", 2024-05-15T16:03:42.695 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw", 2024-05-15T16:03:42.695 INFO:teuthology.orchestra.run.smithi043.stdout: "osd" 2024-05-15T16:03:42.695 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T16:03:42.695 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "15/21 daemons upgraded", 2024-05-15T16:03:42.695 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Currently upgrading iscsi daemons", 2024-05-15T16:03:42.695 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": false 2024-05-15T16:03:42.695 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:03:43.130 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-05-15T16:03:43.130 INFO:teuthology.orchestra.run.smithi043.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-05-15T16:03:43.130 INFO:teuthology.orchestra.run.smithi043.stdout: daemon iscsi.foo.smithi043.aclusg on smithi043 is in unknown state 2024-05-15T16:03:43.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='client.54745 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:43.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:43.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mgr 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mon 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all crash 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all osd 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all mds 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all rgw 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2770830718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.588 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3452421824' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='client.54745 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mgr 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.589 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mon 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all crash 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all osd 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all mds 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all rgw 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T16:03:43.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T16:03:43.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T16:03:43.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2770830718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.591 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:43 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3452421824' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:03:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='client.54745 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:43.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='client.54751 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Target is version 19.0.0-2576-gde5210d5 (squid) 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a'] 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mgr 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mon 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all crash 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all osd 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all mds 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all rgw 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all rbd-mirror 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all cephfs-mirror 2024-05-15T16:03:43.632 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: Upgrade: Setting container_image for all ceph-exporter 2024-05-15T16:03:43.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:43.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2770830718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:03:43.633 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:43 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3452421824' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:03:44.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[82531]: from='client.54763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[82531]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:44.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:44.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[88099]: from='client.54763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:44.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[88099]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:44.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:44.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:44.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:44.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:44 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:44 smithi053 ceph-mon[67220]: from='client.54763 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:03:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:44 smithi053 ceph-mon[67220]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:44 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:44 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:44 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:44.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:44 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:45.585 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:45 smithi043 ceph-mon[88099]: Upgrade: Updating iscsi.foo.smithi043.aclusg 2024-05-15T16:03:45.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:45 smithi043 ceph-mon[88099]: Deploying daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:45.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:45 smithi043 ceph-mon[82531]: Upgrade: Updating iscsi.foo.smithi043.aclusg 2024-05-15T16:03:45.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:45 smithi043 ceph-mon[82531]: Deploying daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:45 smithi053 ceph-mon[67220]: Upgrade: Updating iscsi.foo.smithi043.aclusg 2024-05-15T16:03:45.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:45 smithi053 ceph-mon[67220]: Deploying daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:46.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:46 smithi043 ceph-mon[82531]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-05-15T16:03:46.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:46 smithi043 ceph-mon[88099]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-05-15T16:03:46.551 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:03:46] "GET /metrics HTTP/1.1" 200 38394 "" "Prometheus/2.43.0" 2024-05-15T16:03:46.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:46 smithi053 ceph-mon[67220]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-05-15T16:03:47.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:47 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:47.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:47 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:47.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:47 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:48.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:48 smithi043 ceph-mon[82531]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:03:48.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:48 smithi043 ceph-mon[88099]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:03:48.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:48 smithi053 ceph-mon[67220]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:03:50.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:50 smithi043 ceph-mon[82531]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:50.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:50 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:50.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:50 smithi043 ceph-mon[88099]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:50.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:50 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:50 smithi053 ceph-mon[67220]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:50.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:50 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:03:52.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:52 smithi043 ceph-mon[82531]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:52.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:52 smithi043 ceph-mon[88099]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:52.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:52 smithi053 ceph-mon[67220]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:54.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:54 smithi043 ceph-mon[82531]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:54.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:54 smithi043 ceph-mon[88099]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:54.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:54 smithi053 ceph-mon[67220]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:56.573 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:56 smithi043 ceph-mon[82531]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:56.574 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:56 smithi043 ceph-mon[88099]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:56.574 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:03:56] "GET /metrics HTTP/1.1" 200 38398 "" "Prometheus/2.43.0" 2024-05-15T16:03:56.629 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:56 smithi053 ceph-mon[67220]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:03:57.549 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:03:57 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:03:57.120+0000 7fbf55aa5700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed. 2024-05-15T16:03:57.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[82531]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:57.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:57.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:57.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:57.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:57.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:57.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:57.550 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:57.551 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:57.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[88099]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:57.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:57.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:57.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:57.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:57.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:57.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:57.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:57.551 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:57 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:57 smithi053 ceph-mon[67220]: from='client.25129 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:03:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:57 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:57 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:03:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:57 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:57 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:03:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:57 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:03:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:57 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:57 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi043.aclusg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-05-15T16:03:57.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:57 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:03:58.329 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[82531]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:58.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[82531]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed. 2024-05-15T16:03:58.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[82531]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-05-15T16:03:58.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[82531]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:58.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[82531]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-15T16:03:58.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[82531]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:58.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[82531]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:58.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[82531]: Health check failed: Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-15T16:03:58.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[88099]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:58.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[88099]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed. 2024-05-15T16:03:58.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[88099]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-05-15T16:03:58.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[88099]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:58.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[88099]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-15T16:03:58.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[88099]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:58.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[88099]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:58.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:03:58 smithi043 ceph-mon[88099]: Health check failed: Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-15T16:03:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:58 smithi053 ceph-mon[67220]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:58 smithi053 ceph-mon[67220]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed. 2024-05-15T16:03:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:58 smithi053 ceph-mon[67220]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-05-15T16:03:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:58 smithi053 ceph-mon[67220]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:03:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:58 smithi053 ceph-mon[67220]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-05-15T16:03:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:58 smithi053 ceph-mon[67220]: Reconfiguring iscsi.foo.smithi043.aclusg (dependencies changed)... 2024-05-15T16:03:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:58 smithi053 ceph-mon[67220]: Reconfiguring daemon iscsi.foo.smithi043.aclusg on smithi043 2024-05-15T16:03:58.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:03:58 smithi053 ceph-mon[67220]: Health check failed: Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-05-15T16:04:00.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[82531]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-05-15T16:04:00.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:00.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:00.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T16:04:00.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T16:04:00.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:04:00.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/732558113' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T16:04:00.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[88099]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-05-15T16:04:00.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:00.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:00.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T16:04:00.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T16:04:00.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:04:00.587 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:00 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/732558113' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T16:04:00.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:00 smithi053 ceph-mon[67220]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-05-15T16:04:00.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:00 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:00.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:00 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:00.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:00 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T16:04:00.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:00 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T16:04:00.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:00 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:04:00.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:00 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/732558113' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-15T16:04:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:01 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T16:04:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:01 smithi053 ceph-mon[67220]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T16:04:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:01 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1399710593' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2329311176"}]: dispatch 2024-05-15T16:04:01.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:01 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2329311176"}]: dispatch 2024-05-15T16:04:01.699 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:01 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T16:04:01.699 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:01 smithi043 ceph-mon[82531]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T16:04:01.699 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:01 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1399710593' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2329311176"}]: dispatch 2024-05-15T16:04:01.699 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:01 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2329311176"}]: dispatch 2024-05-15T16:04:01.699 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:01 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-05-15T16:04:01.700 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:01 smithi043 ceph-mon[88099]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-05-15T16:04:01.700 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:01 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1399710593' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2329311176"}]: dispatch 2024-05-15T16:04:01.700 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:01 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2329311176"}]: dispatch 2024-05-15T16:04:02.577 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:02 smithi043 ceph-mon[88099]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-05-15T16:04:02.577 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:02 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2329311176"}]': finished 2024-05-15T16:04:02.577 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:02 smithi043 ceph-mon[88099]: osdmap e149: 8 total, 8 up, 8 in 2024-05-15T16:04:02.577 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:02 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3939622111' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2329311176"}]: dispatch 2024-05-15T16:04:02.577 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:02 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2329311176"}]: dispatch 2024-05-15T16:04:02.577 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:02 smithi043 ceph-mon[82531]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-05-15T16:04:02.577 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:02 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2329311176"}]': finished 2024-05-15T16:04:02.578 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:02 smithi043 ceph-mon[82531]: osdmap e149: 8 total, 8 up, 8 in 2024-05-15T16:04:02.578 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:02 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3939622111' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2329311176"}]: dispatch 2024-05-15T16:04:02.578 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:02 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2329311176"}]: dispatch 2024-05-15T16:04:02.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:02 smithi053 ceph-mon[67220]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-05-15T16:04:02.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:02 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/2329311176"}]': finished 2024-05-15T16:04:02.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:02 smithi053 ceph-mon[67220]: osdmap e149: 8 total, 8 up, 8 in 2024-05-15T16:04:02.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:02 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3939622111' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2329311176"}]: dispatch 2024-05-15T16:04:02.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:02 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2329311176"}]: dispatch 2024-05-15T16:04:03.585 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:03 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2329311176"}]': finished 2024-05-15T16:04:03.585 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:03 smithi043 ceph-mon[88099]: osdmap e150: 8 total, 8 up, 8 in 2024-05-15T16:04:03.586 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:03 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1893483689' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2303034054"}]: dispatch 2024-05-15T16:04:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:03 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2329311176"}]': finished 2024-05-15T16:04:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:03 smithi043 ceph-mon[82531]: osdmap e150: 8 total, 8 up, 8 in 2024-05-15T16:04:03.586 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:03 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1893483689' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2303034054"}]: dispatch 2024-05-15T16:04:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:03 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/2329311176"}]': finished 2024-05-15T16:04:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:03 smithi053 ceph-mon[67220]: osdmap e150: 8 total, 8 up, 8 in 2024-05-15T16:04:03.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:03 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1893483689' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2303034054"}]: dispatch 2024-05-15T16:04:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:04 smithi053 ceph-mon[67220]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-15T16:04:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:04 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1893483689' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2303034054"}]': finished 2024-05-15T16:04:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:04 smithi053 ceph-mon[67220]: osdmap e151: 8 total, 8 up, 8 in 2024-05-15T16:04:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:04 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3838697204' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2300416256"}]: dispatch 2024-05-15T16:04:04.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:04 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2300416256"}]: dispatch 2024-05-15T16:04:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:04 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:04 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:04:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:04:04.631 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:04 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[82531]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-15T16:04:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1893483689' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2303034054"}]': finished 2024-05-15T16:04:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[82531]: osdmap e151: 8 total, 8 up, 8 in 2024-05-15T16:04:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3838697204' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2300416256"}]: dispatch 2024-05-15T16:04:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2300416256"}]: dispatch 2024-05-15T16:04:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:04.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:04:04.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:04:04.837 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:04.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[88099]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-05-15T16:04:04.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1893483689' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2303034054"}]': finished 2024-05-15T16:04:04.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[88099]: osdmap e151: 8 total, 8 up, 8 in 2024-05-15T16:04:04.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3838697204' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2300416256"}]: dispatch 2024-05-15T16:04:04.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2300416256"}]: dispatch 2024-05-15T16:04:04.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:04.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:04.837 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:04:04.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:04:04.838 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:04 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:05 smithi053 ceph-mon[67220]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 659 B/s rd, 0 op/s 2024-05-15T16:04:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:05 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2300416256"}]': finished 2024-05-15T16:04:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:05 smithi053 ceph-mon[67220]: osdmap e152: 8 total, 8 up, 8 in 2024-05-15T16:04:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:05 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:05 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:05 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1452848949' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/402692770"}]: dispatch 2024-05-15T16:04:05.630 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:05 smithi053 ceph-mon[67220]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-15T16:04:05.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[82531]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 659 B/s rd, 0 op/s 2024-05-15T16:04:05.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2300416256"}]': finished 2024-05-15T16:04:05.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[82531]: osdmap e152: 8 total, 8 up, 8 in 2024-05-15T16:04:05.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:05.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:05.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1452848949' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/402692770"}]: dispatch 2024-05-15T16:04:05.667 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[82531]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-15T16:04:05.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[88099]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 659 B/s rd, 0 op/s 2024-05-15T16:04:05.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/2300416256"}]': finished 2024-05-15T16:04:05.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[88099]: osdmap e152: 8 total, 8 up, 8 in 2024-05-15T16:04:05.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:05.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:05.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1452848949' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/402692770"}]: dispatch 2024-05-15T16:04:05.668 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:05 smithi043 ceph-mon[88099]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-05-15T16:04:06.653 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:04:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:04:06] "GET /metrics HTTP/1.1" 200 38478 "" "Prometheus/2.43.0" 2024-05-15T16:04:07.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:06 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1452848949' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/402692770"}]': finished 2024-05-15T16:04:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:06 smithi043 ceph-mon[82531]: osdmap e153: 8 total, 8 up, 8 in 2024-05-15T16:04:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:06 smithi043 ceph-mon[82531]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:04:07.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:06 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1741602184' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/886772044"}]: dispatch 2024-05-15T16:04:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:06 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1452848949' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/402692770"}]': finished 2024-05-15T16:04:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:06 smithi043 ceph-mon[88099]: osdmap e153: 8 total, 8 up, 8 in 2024-05-15T16:04:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:06 smithi043 ceph-mon[88099]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:04:07.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:06 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1741602184' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/886772044"}]: dispatch 2024-05-15T16:04:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:06 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1452848949' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/402692770"}]': finished 2024-05-15T16:04:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:06 smithi053 ceph-mon[67220]: osdmap e153: 8 total, 8 up, 8 in 2024-05-15T16:04:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:06 smithi053 ceph-mon[67220]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-05-15T16:04:07.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:06 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1741602184' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/886772044"}]: dispatch 2024-05-15T16:04:08.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:07 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1741602184' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/886772044"}]': finished 2024-05-15T16:04:08.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:07 smithi043 ceph-mon[82531]: osdmap e154: 8 total, 8 up, 8 in 2024-05-15T16:04:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:07 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1678425927' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1211703430"}]: dispatch 2024-05-15T16:04:08.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:07 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1211703430"}]: dispatch 2024-05-15T16:04:08.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:07 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1741602184' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/886772044"}]': finished 2024-05-15T16:04:08.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:07 smithi043 ceph-mon[88099]: osdmap e154: 8 total, 8 up, 8 in 2024-05-15T16:04:08.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:07 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1678425927' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1211703430"}]: dispatch 2024-05-15T16:04:08.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:07 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1211703430"}]: dispatch 2024-05-15T16:04:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:07 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1741602184' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/886772044"}]': finished 2024-05-15T16:04:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:07 smithi053 ceph-mon[67220]: osdmap e154: 8 total, 8 up, 8 in 2024-05-15T16:04:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:07 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1678425927' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1211703430"}]: dispatch 2024-05-15T16:04:08.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:07 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1211703430"}]: dispatch 2024-05-15T16:04:09.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:08 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1211703430"}]': finished 2024-05-15T16:04:09.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:08 smithi043 ceph-mon[82531]: osdmap e155: 8 total, 8 up, 8 in 2024-05-15T16:04:09.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:08 smithi043 ceph-mon[82531]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:09.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:08 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2527845214' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3059089968"}]: dispatch 2024-05-15T16:04:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:08 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1211703430"}]': finished 2024-05-15T16:04:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:08 smithi043 ceph-mon[88099]: osdmap e155: 8 total, 8 up, 8 in 2024-05-15T16:04:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:08 smithi043 ceph-mon[88099]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:09.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:08 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2527845214' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3059089968"}]: dispatch 2024-05-15T16:04:09.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:08 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/1211703430"}]': finished 2024-05-15T16:04:09.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:08 smithi053 ceph-mon[67220]: osdmap e155: 8 total, 8 up, 8 in 2024-05-15T16:04:09.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:08 smithi053 ceph-mon[67220]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:09.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:08 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2527845214' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3059089968"}]: dispatch 2024-05-15T16:04:10.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:09 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2527845214' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3059089968"}]': finished 2024-05-15T16:04:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:09 smithi043 ceph-mon[82531]: osdmap e156: 8 total, 8 up, 8 in 2024-05-15T16:04:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:09 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/910158522' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/145318613"}]: dispatch 2024-05-15T16:04:10.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:09 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/145318613"}]: dispatch 2024-05-15T16:04:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:09 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2527845214' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3059089968"}]': finished 2024-05-15T16:04:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:09 smithi043 ceph-mon[88099]: osdmap e156: 8 total, 8 up, 8 in 2024-05-15T16:04:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:09 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/910158522' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/145318613"}]: dispatch 2024-05-15T16:04:10.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:09 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/145318613"}]: dispatch 2024-05-15T16:04:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:09 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2527845214' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/3059089968"}]': finished 2024-05-15T16:04:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:09 smithi053 ceph-mon[67220]: osdmap e156: 8 total, 8 up, 8 in 2024-05-15T16:04:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:09 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/910158522' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/145318613"}]: dispatch 2024-05-15T16:04:10.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:09 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/145318613"}]: dispatch 2024-05-15T16:04:11.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/145318613"}]': finished 2024-05-15T16:04:11.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[82531]: osdmap e157: 8 total, 8 up, 8 in 2024-05-15T16:04:11.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[82531]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:11.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:11.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3562771282' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/1923093507"}]: dispatch 2024-05-15T16:04:11.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/1923093507"}]: dispatch 2024-05-15T16:04:11.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/145318613"}]': finished 2024-05-15T16:04:11.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[88099]: osdmap e157: 8 total, 8 up, 8 in 2024-05-15T16:04:11.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[88099]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:11.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:11.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3562771282' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/1923093507"}]: dispatch 2024-05-15T16:04:11.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:10 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/1923093507"}]: dispatch 2024-05-15T16:04:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:10 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/145318613"}]': finished 2024-05-15T16:04:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:10 smithi053 ceph-mon[67220]: osdmap e157: 8 total, 8 up, 8 in 2024-05-15T16:04:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:10 smithi053 ceph-mon[67220]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:10 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:10 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3562771282' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/1923093507"}]: dispatch 2024-05-15T16:04:11.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:10 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/1923093507"}]: dispatch 2024-05-15T16:04:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:11 smithi043 ceph-mon[82531]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/1923093507"}]': finished 2024-05-15T16:04:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:11 smithi043 ceph-mon[82531]: osdmap e158: 8 total, 8 up, 8 in 2024-05-15T16:04:12.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:11 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2041057513' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/1923093507"}]: dispatch 2024-05-15T16:04:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:11 smithi043 ceph-mon[88099]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/1923093507"}]': finished 2024-05-15T16:04:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:11 smithi043 ceph-mon[88099]: osdmap e158: 8 total, 8 up, 8 in 2024-05-15T16:04:12.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:11 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2041057513' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/1923093507"}]: dispatch 2024-05-15T16:04:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:11 smithi053 ceph-mon[67220]: from='client.? ' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6800/1923093507"}]': finished 2024-05-15T16:04:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:11 smithi053 ceph-mon[67220]: osdmap e158: 8 total, 8 up, 8 in 2024-05-15T16:04:12.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:11 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2041057513' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/1923093507"}]: dispatch 2024-05-15T16:04:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:12 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2041057513' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/1923093507"}]': finished 2024-05-15T16:04:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:12 smithi043 ceph-mon[82531]: osdmap e159: 8 total, 8 up, 8 in 2024-05-15T16:04:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:12 smithi043 ceph-mon[82531]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:13.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:12 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3611882758' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/578044507"}]: dispatch 2024-05-15T16:04:13.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:12 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2041057513' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/1923093507"}]': finished 2024-05-15T16:04:13.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:12 smithi043 ceph-mon[88099]: osdmap e159: 8 total, 8 up, 8 in 2024-05-15T16:04:13.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:12 smithi043 ceph-mon[88099]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:13.087 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:12 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3611882758' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/578044507"}]: dispatch 2024-05-15T16:04:13.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:12 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2041057513' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:6801/1923093507"}]': finished 2024-05-15T16:04:13.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:12 smithi053 ceph-mon[67220]: osdmap e159: 8 total, 8 up, 8 in 2024-05-15T16:04:13.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:12 smithi053 ceph-mon[67220]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:13.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:12 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3611882758' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/578044507"}]: dispatch 2024-05-15T16:04:13.507 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T16:04:13.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:13 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/3611882758' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/578044507"}]': finished 2024-05-15T16:04:13.835 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:13 smithi043 ceph-mon[88099]: osdmap e160: 8 total, 8 up, 8 in 2024-05-15T16:04:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:13 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/3611882758' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/578044507"}]': finished 2024-05-15T16:04:13.836 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:13 smithi043 ceph-mon[82531]: osdmap e160: 8 total, 8 up, 8 in 2024-05-15T16:04:13.856 INFO:teuthology.orchestra.run.smithi043.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed." 2024-05-15T16:04:14.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:13 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/3611882758' entity='client.iscsi.foo.smithi043.aclusg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.43:0/578044507"}]': finished 2024-05-15T16:04:14.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:13 smithi053 ceph-mon[67220]: osdmap e160: 8 total, 8 up, 8 in 2024-05-15T16:04:14.792 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T16:04:15.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:14 smithi043 ceph-mon[82531]: from='client.54871 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:04:15.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:14 smithi043 ceph-mon[82531]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:15.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:14 smithi043 ceph-mon[82531]: from='client.44934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:04:15.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:14 smithi043 ceph-mon[88099]: from='client.54871 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:04:15.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:14 smithi043 ceph-mon[88099]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:15.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:14 smithi043 ceph-mon[88099]: from='client.44934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:04:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:14 smithi053 ceph-mon[67220]: from='client.54871 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:04:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:14 smithi053 ceph-mon[67220]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:15.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:14 smithi053 ceph-mon[67220]: from='client.44934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:04:16.556 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:04:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:04:16] "GET /metrics HTTP/1.1" 200 38478 "" "Prometheus/2.43.0" 2024-05-15T16:04:17.021 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T16:04:17.021 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (11m) 13s ago 33m - - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T16:04:17.021 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (10m) 69s ago 33m 83.0M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T16:04:17.021 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 running (17s) 13s ago 32m - - 3.7 48bc18eb22d8 a4a3dde42173 2024-05-15T16:04:17.021 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (12m) 69s ago 37m 455M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T16:04:17.021 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (22m) 13s ago 39m - - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (8m) 13s ago 39m - 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (9m) 69s ago 38m 56.5M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (8m) 13s ago 38m - 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (12m) 13s ago 34m - - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (12m) 69s ago 34m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (6m) 13s ago 37m - 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (6m) 13s ago 37m - 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (5m) 13s ago 36m - 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (4m) 13s ago 36m - 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 258344f90360 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (3m) 69s ago 36m 59.3M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 b342d0fab575 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (3m) 69s ago 35m 57.4M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 f709584ed028 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (2m) 69s ago 35m 47.8M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 d5353ef72311 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (2m) 69s ago 34m 55.0M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 784f18f73ebd 2024-05-15T16:04:17.022 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (11m) 69s ago 34m 79.7M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T16:04:17.023 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (79s) 13s ago 32m - - 19.0.0-2576-gde5210d5 48bc18eb22d8 77771618aae7 2024-05-15T16:04:17.023 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (73s) 69s ago 32m 74.9M - 19.0.0-2576-gde5210d5 48bc18eb22d8 64952a4875cd 2024-05-15T16:04:17.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:16 smithi053 ceph-mon[67220]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:17.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:16 smithi043 ceph-mon[82531]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:16 smithi043 ceph-mon[88099]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:17.626 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions' 2024-05-15T16:04:18.129 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:17 smithi043 ceph-mon[82531]: from='client.54883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:04:18.129 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:17 smithi043 ceph-mon[88099]: from='client.54883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:04:18.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:17 smithi053 ceph-mon[67220]: from='client.54883 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:04:19.109 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:18 smithi043 ceph-mon[82531]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:04:19.109 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:18 smithi043 ceph-mon[88099]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:04:19.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:18 smithi053 ceph-mon[67220]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-05-15T16:04:19.645 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:04:19.645 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T16:04:19.645 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T16:04:19.645 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:04:19.645 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T16:04:19.645 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:04:19.645 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:04:19.645 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T16:04:19.645 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 8 2024-05-15T16:04:19.645 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:04:19.646 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T16:04:19.646 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:04:19.646 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:04:19.646 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T16:04:19.646 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 15 2024-05-15T16:04:19.646 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T16:04:19.646 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:04:19.965 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:19 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2197216192' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:04:19.965 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:19 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:19.966 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:19 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:19.966 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:19 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2197216192' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:04:19.966 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:19 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:19.966 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:19 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:19 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2197216192' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:04:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:19 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:04:20.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:19 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:20.341 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-05-15T16:04:20.944 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:20 smithi043 ceph-mon[82531]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-05-15T16:04:20.944 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:20 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:20.945 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:20 smithi043 ceph-mon[88099]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-05-15T16:04:20.945 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:20 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:21.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:20 smithi053 ceph-mon[67220]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-05-15T16:04:21.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:20 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:22.060 INFO:teuthology.orchestra.run.smithi043.stdout:wait for servicemap items w/ changing names to refresh 2024-05-15T16:04:22.837 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'sleep 60' 2024-05-15T16:04:23.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:22 smithi043 ceph-mon[82531]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:04:23.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:22 smithi043 ceph-mon[88099]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:04:23.129 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:22 smithi053 ceph-mon[67220]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-05-15T16:04:25.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:24 smithi053 ceph-mon[67220]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-15T16:04:25.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:24 smithi043 ceph-mon[82531]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-15T16:04:25.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:24 smithi043 ceph-mon[88099]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-05-15T16:04:26.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:04:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:04:26] "GET /metrics HTTP/1.1" 200 38473 "" "Prometheus/2.43.0" 2024-05-15T16:04:27.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:26 smithi053 ceph-mon[67220]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:27.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:26 smithi043 ceph-mon[82531]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:27.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:26 smithi043 ceph-mon[88099]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:29.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:28 smithi053 ceph-mon[67220]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:29.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:28 smithi043 ceph-mon[82531]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:29.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:28 smithi043 ceph-mon[88099]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:30 smithi053 ceph-mon[67220]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:31.130 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:30 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:31.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:30 smithi043 ceph-mon[82531]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:31.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:30 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:30 smithi043 ceph-mon[88099]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:31.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:30 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:33.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:32 smithi043 ceph-mon[82531]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:33.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:32 smithi043 ceph-mon[88099]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:33.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:32 smithi053 ceph-mon[67220]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:35.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:34 smithi043 ceph-mon[82531]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:35.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:34 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:35.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:34 smithi043 ceph-mon[88099]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:35.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:34 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:34 smithi053 ceph-mon[67220]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:34 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:36.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:04:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:04:36] "GET /metrics HTTP/1.1" 200 38477 "" "Prometheus/2.43.0" 2024-05-15T16:04:37.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:36 smithi043 ceph-mon[82531]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:37.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:36 smithi043 ceph-mon[88099]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:37.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:36 smithi053 ceph-mon[67220]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:39.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:38 smithi043 ceph-mon[82531]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:39.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:38 smithi043 ceph-mon[88099]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:39.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:38 smithi053 ceph-mon[67220]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:41.335 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:40 smithi043 ceph-mon[88099]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:41.335 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:40 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:41.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:40 smithi043 ceph-mon[82531]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:41.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:40 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:41.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:40 smithi053 ceph-mon[67220]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:41.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:40 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:43.335 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:42 smithi043 ceph-mon[88099]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:43.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:42 smithi043 ceph-mon[82531]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:42 smithi053 ceph-mon[67220]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:45.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:44 smithi043 ceph-mon[88099]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:45.338 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:44 smithi043 ceph-mon[82531]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:45.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:44 smithi053 ceph-mon[67220]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:46.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:04:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:04:46] "GET /metrics HTTP/1.1" 200 38477 "" "Prometheus/2.43.0" 2024-05-15T16:04:47.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:46 smithi043 ceph-mon[82531]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:47.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:46 smithi043 ceph-mon[88099]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:47.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:46 smithi053 ceph-mon[67220]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:49.181 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:48 smithi053 ceph-mon[67220]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:49.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:48 smithi043 ceph-mon[82531]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:49.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:48 smithi043 ceph-mon[88099]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:50.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:49 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:50.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:49 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:50.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:49 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:04:51.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:50 smithi043 ceph-mon[88099]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:51.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:50 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:51.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:50 smithi043 ceph-mon[82531]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:51.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:50 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:51.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:50 smithi053 ceph-mon[67220]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:51.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:50 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:04:53.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:52 smithi043 ceph-mon[82531]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:53.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:52 smithi043 ceph-mon[88099]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:53.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:52 smithi053 ceph-mon[67220]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:55.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:54 smithi043 ceph-mon[82531]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:55.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:54 smithi043 ceph-mon[88099]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:55.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:54 smithi053 ceph-mon[67220]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:56.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:04:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:04:56] "GET /metrics HTTP/1.1" 200 38476 "" "Prometheus/2.43.0" 2024-05-15T16:04:57.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:56 smithi043 ceph-mon[82531]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:57.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:56 smithi043 ceph-mon[88099]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:57.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:56 smithi053 ceph-mon[67220]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:04:59.335 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:04:58 smithi043 ceph-mon[88099]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:59.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:04:58 smithi043 ceph-mon[82531]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:04:59.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:04:58 smithi053 ceph-mon[67220]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:01.335 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:00 smithi043 ceph-mon[88099]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:01.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:00 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:01.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:00 smithi043 ceph-mon[82531]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:01.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:00 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:01.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:00 smithi053 ceph-mon[67220]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:01.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:00 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:03.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:02 smithi043 ceph-mon[82531]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:03.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:02 smithi043 ceph-mon[88099]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:03.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:02 smithi053 ceph-mon[67220]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:05.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[82531]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:05.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:05:05.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:05:05.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:05:05.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[82531]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:05:05.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:05.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[88099]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:05.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:05:05.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:05:05.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:05:05.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[88099]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:05:05.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:04 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:05.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:04 smithi053 ceph-mon[67220]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:05.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-15T16:05:05.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-15T16:05:05.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-15T16:05:05.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:04 smithi053 ceph-mon[67220]: from='mgr.44106 ' entity='mgr.y' 2024-05-15T16:05:05.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:04 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:06.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:06 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:05:06] "GET /metrics HTTP/1.1" 200 38475 "" "Prometheus/2.43.0" 2024-05-15T16:05:07.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:06 smithi043 ceph-mon[82531]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:07.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:06 smithi043 ceph-mon[88099]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:07.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:06 smithi053 ceph-mon[67220]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:09.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:08 smithi043 ceph-mon[82531]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:09.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:08 smithi043 ceph-mon[88099]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:09.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:08 smithi053 ceph-mon[67220]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:11.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:10 smithi043 ceph-mon[82531]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:11.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:10 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:11.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:10 smithi043 ceph-mon[88099]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:11.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:10 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:11.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:10 smithi053 ceph-mon[67220]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:11.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:10 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:13.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:12 smithi043 ceph-mon[82531]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:13.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:12 smithi043 ceph-mon[88099]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:13.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:12 smithi053 ceph-mon[67220]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:15.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:14 smithi043 ceph-mon[82531]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:15.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:14 smithi043 ceph-mon[88099]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:15.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:14 smithi053 ceph-mon[67220]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:16.835 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:16 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:05:16] "GET /metrics HTTP/1.1" 200 38475 "" "Prometheus/2.43.0" 2024-05-15T16:05:17.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:16 smithi043 ceph-mon[82531]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:17.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:16 smithi043 ceph-mon[88099]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:17.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:16 smithi053 ceph-mon[67220]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:19.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:18 smithi043 ceph-mon[82531]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:19.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:18 smithi043 ceph-mon[88099]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:19.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:18 smithi053 ceph-mon[67220]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:20.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:19 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:20.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:19 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:20.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:19 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:21.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:20 smithi043 ceph-mon[82531]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:21.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:20 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:21.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:20 smithi043 ceph-mon[88099]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:21.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:20 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:21.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:20 smithi053 ceph-mon[67220]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:21.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:20 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:23.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:22 smithi043 ceph-mon[82531]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:23.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:22 smithi043 ceph-mon[88099]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:23.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:22 smithi053 ceph-mon[67220]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:25.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:24 smithi043 ceph-mon[82531]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:25.337 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:24 smithi043 ceph-mon[88099]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:25.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:24 smithi053 ceph-mon[67220]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-05-15T16:05:25.448 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ps' 2024-05-15T16:05:26.431 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:05:26] "GET /metrics HTTP/1.1" 200 38477 "" "Prometheus/2.43.0" 2024-05-15T16:05:27.077 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:26 smithi043 ceph-mon[82531]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:27.077 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:26 smithi043 ceph-mon[88099]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:27.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:26 smithi053 ceph-mon[67220]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:27.527 INFO:teuthology.orchestra.run.smithi043.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-15T16:05:27.527 INFO:teuthology.orchestra.run.smithi043.stdout:alertmanager.a smithi043 *:9093,9094 running (12m) 83s ago 34m - - 0.25.0 c8568f914cd2 bd365acb01c5 2024-05-15T16:05:27.527 INFO:teuthology.orchestra.run.smithi043.stdout:grafana.a smithi053 *:3000 running (12m) 2m ago 34m 83.0M - 9.4.12 2bacad6d85d8 16402042b8e5 2024-05-15T16:05:27.528 INFO:teuthology.orchestra.run.smithi043.stdout:iscsi.foo.smithi043.aclusg smithi043 running (88s) 83s ago 33m - - 3.7 48bc18eb22d8 a4a3dde42173 2024-05-15T16:05:27.528 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.x smithi053 *:8443,9283,8765 running (13m) 2m ago 39m 455M - 19.0.0-2576-gde5210d5 48bc18eb22d8 006c00101d67 2024-05-15T16:05:27.528 INFO:teuthology.orchestra.run.smithi043.stdout:mgr.y smithi043 *:8443,9283,8765 running (23m) 83s ago 40m - - 19.0.0-2576-gde5210d5 48bc18eb22d8 8de8e1ec414e 2024-05-15T16:05:27.528 INFO:teuthology.orchestra.run.smithi043.stdout:mon.a smithi043 running (10m) 83s ago 40m - 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 c488ab1aa87e 2024-05-15T16:05:27.528 INFO:teuthology.orchestra.run.smithi043.stdout:mon.b smithi053 running (10m) 2m ago 39m 56.5M 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 0bcacc448823 2024-05-15T16:05:27.528 INFO:teuthology.orchestra.run.smithi043.stdout:mon.c smithi043 running (9m) 83s ago 39m - 2048M 19.0.0-2576-gde5210d5 48bc18eb22d8 d4da75e1a85c 2024-05-15T16:05:27.529 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.a smithi043 *:9100 running (13m) 83s ago 35m - - 1.5.0 0da6a335fe13 c92a2c2e9bf7 2024-05-15T16:05:27.529 INFO:teuthology.orchestra.run.smithi043.stdout:node-exporter.b smithi053 *:9100 running (13m) 2m ago 35m 16.7M - 1.5.0 0da6a335fe13 1fb31435f151 2024-05-15T16:05:27.529 INFO:teuthology.orchestra.run.smithi043.stdout:osd.0 smithi043 running (7m) 83s ago 38m - 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 d646f037d33c 2024-05-15T16:05:27.529 INFO:teuthology.orchestra.run.smithi043.stdout:osd.1 smithi043 running (7m) 83s ago 38m - 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 9bdd94dfda49 2024-05-15T16:05:27.529 INFO:teuthology.orchestra.run.smithi043.stdout:osd.2 smithi043 running (6m) 83s ago 38m - 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 087a14715d56 2024-05-15T16:05:27.529 INFO:teuthology.orchestra.run.smithi043.stdout:osd.3 smithi043 running (5m) 83s ago 37m - 2993M 19.0.0-2576-gde5210d5 48bc18eb22d8 258344f90360 2024-05-15T16:05:27.529 INFO:teuthology.orchestra.run.smithi043.stdout:osd.4 smithi053 running (5m) 2m ago 37m 59.3M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 b342d0fab575 2024-05-15T16:05:27.529 INFO:teuthology.orchestra.run.smithi043.stdout:osd.5 smithi053 running (4m) 2m ago 36m 57.4M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 f709584ed028 2024-05-15T16:05:27.529 INFO:teuthology.orchestra.run.smithi043.stdout:osd.6 smithi053 running (3m) 2m ago 36m 47.8M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 d5353ef72311 2024-05-15T16:05:27.529 INFO:teuthology.orchestra.run.smithi043.stdout:osd.7 smithi053 running (3m) 2m ago 36m 55.0M 3249M 19.0.0-2576-gde5210d5 48bc18eb22d8 784f18f73ebd 2024-05-15T16:05:27.530 INFO:teuthology.orchestra.run.smithi043.stdout:prometheus.a smithi053 *:9095 running (12m) 2m ago 35m 79.7M - 2.43.0 a07b618ecd1d c420668ab640 2024-05-15T16:05:27.530 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi043.fiueec smithi043 *:8000 running (2m) 83s ago 33m - - 19.0.0-2576-gde5210d5 48bc18eb22d8 77771618aae7 2024-05-15T16:05:27.530 INFO:teuthology.orchestra.run.smithi043.stdout:rgw.foo.smithi053.vjmwpu smithi053 *:8000 running (2m) 2m ago 33m 74.9M - 19.0.0-2576-gde5210d5 48bc18eb22d8 64952a4875cd 2024-05-15T16:05:28.114 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions' 2024-05-15T16:05:29.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:28 smithi043 ceph-mon[88099]: from='client.54895 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:29.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:28 smithi043 ceph-mon[88099]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:29.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:28 smithi043 ceph-mon[82531]: from='client.54895 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:29.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:28 smithi043 ceph-mon[82531]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:28 smithi053 ceph-mon[67220]: from='client.54895 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:29.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:28 smithi053 ceph-mon[67220]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:30.433 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:05:30.433 INFO:teuthology.orchestra.run.smithi043.stdout: "mon": { 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 3 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr": { 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: "osd": { 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 8 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw": { 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 2 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: }, 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: "overall": { 2024-05-15T16:05:30.434 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)": 15 2024-05-15T16:05:30.435 INFO:teuthology.orchestra.run.smithi043.stdout: } 2024-05-15T16:05:30.435 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:05:31.046 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:30 smithi043 ceph-mon[82531]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:31.046 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:30 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:31.046 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:30 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/66225762' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:05:31.047 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:30 smithi043 ceph-mon[88099]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:31.047 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:30 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:31.047 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:30 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/66225762' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:05:31.074 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch upgrade status' 2024-05-15T16:05:31.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:30 smithi053 ceph-mon[67220]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:31.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:30 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:31.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:30 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/66225762' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:05:33.098 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:05:33.098 INFO:teuthology.orchestra.run.smithi043.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a", 2024-05-15T16:05:33.098 INFO:teuthology.orchestra.run.smithi043.stdout: "in_progress": true, 2024-05-15T16:05:33.098 INFO:teuthology.orchestra.run.smithi043.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-15T16:05:33.098 INFO:teuthology.orchestra.run.smithi043.stdout: "services_complete": [ 2024-05-15T16:05:33.098 INFO:teuthology.orchestra.run.smithi043.stdout: "mgr", 2024-05-15T16:05:33.098 INFO:teuthology.orchestra.run.smithi043.stdout: "rgw", 2024-05-15T16:05:33.098 INFO:teuthology.orchestra.run.smithi043.stdout: "iscsi", 2024-05-15T16:05:33.098 INFO:teuthology.orchestra.run.smithi043.stdout: "mon", 2024-05-15T16:05:33.099 INFO:teuthology.orchestra.run.smithi043.stdout: "osd" 2024-05-15T16:05:33.099 INFO:teuthology.orchestra.run.smithi043.stdout: ], 2024-05-15T16:05:33.099 INFO:teuthology.orchestra.run.smithi043.stdout: "progress": "16/21 daemons upgraded", 2024-05-15T16:05:33.099 INFO:teuthology.orchestra.run.smithi043.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed.", 2024-05-15T16:05:33.099 INFO:teuthology.orchestra.run.smithi043.stdout: "is_paused": true 2024-05-15T16:05:33.099 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:05:33.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:32 smithi043 ceph-mon[82531]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:33.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:32 smithi043 ceph-mon[88099]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:33.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:32 smithi053 ceph-mon[67220]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:33.765 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph health detail' 2024-05-15T16:05:34.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:33 smithi043 ceph-mon[82531]: from='client.54907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:34.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:33 smithi043 ceph-mon[88099]: from='client.54907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:34.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:33 smithi053 ceph-mon[67220]: from='client.54907 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:35.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:34 smithi043 ceph-mon[82531]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:35.211 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:34 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:35.211 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:34 smithi043 ceph-mon[88099]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:35.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:34 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:34 smithi053 ceph-mon[67220]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:35.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:34 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:35.738 INFO:teuthology.orchestra.run.smithi043.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed. 2024-05-15T16:05:35.738 INFO:teuthology.orchestra.run.smithi043.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi043.aclusg on host smithi043 failed. 2024-05-15T16:05:35.738 INFO:teuthology.orchestra.run.smithi043.stdout: Upgrade daemon: iscsi.foo.smithi043.aclusg: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi043.aclusg ... 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout:Creating ceph-iscsi config... 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout:Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/iscsi-gateway.cfg 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout:Write file: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/iscsi.foo.smithi043.aclusg/tcmu-runner-entrypoint.sh 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2d22d82864\x2d12cf\x2d11ef\x2dbc98\x2dc7b262605968.slice/ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout:Non-zero exit code 1 from systemctl start ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout:systemctl: stderr Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout:systemctl: stderr See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout:Traceback (most recent call last): 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout: return _run_code(code, main_globals, None, 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout: exec(code, run_globals) 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5618, in 2024-05-15T16:05:35.739 INFO:teuthology.orchestra.run.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 5606, in main 2024-05-15T16:05:35.740 INFO:teuthology.orchestra.run.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3096, in command_deploy_from 2024-05-15T16:05:35.740 INFO:teuthology.orchestra.run.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3131, in _common_deploy 2024-05-15T16:05:35.740 INFO:teuthology.orchestra.run.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 3151, in _deploy_daemon_container 2024-05-15T16:05:35.740 INFO:teuthology.orchestra.run.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1087, in deploy_daemon 2024-05-15T16:05:35.740 INFO:teuthology.orchestra.run.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/__main__.py", line 1223, in deploy_daemon_units 2024-05-15T16:05:35.740 INFO:teuthology.orchestra.run.smithi043.stdout: File "/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/cephadm.0274d6ff5b9f2aae0b3541aacf95498bc9ef10781b3c093549ab0705b6baf2bc/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-05-15T16:05:35.740 INFO:teuthology.orchestra.run.smithi043.stdout:RuntimeError: Failed command: systemctl start ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg: Job for ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service failed because the control process exited with error code. 2024-05-15T16:05:35.740 INFO:teuthology.orchestra.run.smithi043.stdout:See "systemctl status ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" and "journalctl -xeu ceph-22d82864-12cf-11ef-bc98-c7b262605968@iscsi.foo.smithi043.aclusg.service" for details. 2024-05-15T16:05:36.085 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:35 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/4058763857' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:05:36.085 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:35 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/4058763857' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:05:36.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:35 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/4058763857' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-15T16:05:36.433 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:36 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:05:36] "GET /metrics HTTP/1.1" 200 38474 "" "Prometheus/2.43.0" 2024-05-15T16:05:36.459 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-05-15T16:05:37.078 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:36 smithi043 ceph-mon[82531]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:37.079 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:36 smithi043 ceph-mon[88099]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:37.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:36 smithi053 ceph-mon[67220]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:38.504 INFO:teuthology.orchestra.run.smithi043.stdout:true 2024-05-15T16:05:39.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:38 smithi043 ceph-mon[82531]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:39.086 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:38 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/2544339321' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:05:39.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:38 smithi043 ceph-mon[88099]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:39.086 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:38 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/2544339321' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:05:39.128 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-05-15T16:05:39.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:38 smithi053 ceph-mon[67220]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:39.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:38 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/2544339321' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:05:41.141 INFO:teuthology.orchestra.run.smithi043.stdout: "ceph version 19.0.0-2576-gde5210d5 (de5210d58560a4684fdb5b238ef3e6e0285a1de8) squid (dev)" 2024-05-15T16:05:41.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:40 smithi043 ceph-mon[82531]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:41.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:40 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:41.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:40 smithi043 ceph-mon[88099]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:41.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:40 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:41.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:40 smithi053 ceph-mon[67220]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:41.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:40 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:41.724 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -e sha1=de5210d58560a4684fdb5b238ef3e6e0285a1de8 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-05-15T16:05:42.323 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:41 smithi043 ceph-mon[82531]: from='client.? 172.21.15.43:0/1943709294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:05:42.324 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:41 smithi043 ceph-mon[88099]: from='client.? 172.21.15.43:0/1943709294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:05:42.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:41 smithi053 ceph-mon[67220]: from='client.? 172.21.15.43:0/1943709294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-15T16:05:43.215 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:42 smithi043 ceph-mon[82531]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:43.215 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:42 smithi043 ceph-mon[88099]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:43.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:42 smithi053 ceph-mon[67220]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:43.693 INFO:teuthology.orchestra.run.smithi043.stdout:osd 8 2m ago - 2024-05-15T16:05:44.446 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-15T16:05:44.464 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi043.front.sepia.ceph.com 2024-05-15T16:05:44.464 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-05-15T16:05:45.081 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:44 smithi043 ceph-mon[82531]: from='client.44988 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:45.081 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:45 smithi043 ceph-mon[82531]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:45.081 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:44 smithi043 ceph-mon[88099]: from='client.44988 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:45.081 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:45 smithi043 ceph-mon[88099]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:45.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:45 smithi053 ceph-mon[67220]: from='client.44988 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:45.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:45 smithi053 ceph-mon[67220]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:46.836 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:46 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:05:46] "GET /metrics HTTP/1.1" 200 38474 "" "Prometheus/2.43.0" 2024-05-15T16:05:47.102 INFO:teuthology.orchestra.run.smithi043.stdout:{ 2024-05-15T16:05:47.102 INFO:teuthology.orchestra.run.smithi043.stdout: "image": "quay.io/ceph/ceph", 2024-05-15T16:05:47.102 INFO:teuthology.orchestra.run.smithi043.stdout: "registry": "quay.io", 2024-05-15T16:05:47.102 INFO:teuthology.orchestra.run.smithi043.stdout: "bare_image": "ceph/ceph", 2024-05-15T16:05:47.102 INFO:teuthology.orchestra.run.smithi043.stdout: "versions": [] 2024-05-15T16:05:47.102 INFO:teuthology.orchestra.run.smithi043.stdout:} 2024-05-15T16:05:47.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:47 smithi043 ceph-mon[82531]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:47.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:47 smithi043 ceph-mon[88099]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:47.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:47 smithi053 ceph-mon[67220]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:47.704 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-05-15T16:05:48.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:48 smithi043 ceph-mon[82531]: from='client.54937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:48.336 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:48 smithi043 ceph-mon[88099]: from='client.54937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:48.379 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:48 smithi053 ceph-mon[67220]: from='client.54937 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:49.281 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:49 smithi053 ceph-mon[67220]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:49.311 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:49 smithi043 ceph-mon[82531]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:49.311 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:49 smithi043 ceph-mon[88099]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:50.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:50 smithi043 ceph-mon[82531]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:50.335 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:50 smithi043 ceph-mon[88099]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:50.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:50 smithi053 ceph-mon[67220]: from='mgr.44106 172.21.15.43:0/1404179398' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-15T16:05:51.293 INFO:teuthology.orchestra.run.smithi043.stdout: "16.2.0", 2024-05-15T16:05:51.293 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:51 smithi043 ceph-mon[82531]: from='client.45000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:51.293 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:51 smithi043 ceph-mon[82531]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:51.293 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:51 smithi043 ceph-mon[82531]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:51.293 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:51 smithi043 ceph-mon[88099]: from='client.45000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:51.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:51 smithi043 ceph-mon[88099]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:51.294 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:51 smithi043 ceph-mon[88099]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:51.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:51 smithi053 ceph-mon[67220]: from='client.45000 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:51.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:51 smithi053 ceph-mon[67220]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:51.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:51 smithi053 ceph-mon[67220]: from='client.44844 -' entity='client.iscsi.foo.smithi043.aclusg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-05-15T16:05:51.887 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 22d82864-12cf-11ef-bc98-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-05-15T16:05:53.179 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:53 smithi043 ceph-mon[88099]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:53.180 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:53 smithi043 ceph-mon[82531]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:53.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:53 smithi053 ceph-mon[67220]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-05-15T16:05:54.652 INFO:teuthology.orchestra.run.smithi043.stdout: "v16.2.2", 2024-05-15T16:05:54.653 INFO:teuthology.orchestra.run.smithi043.stdout: "v16.2.2-20210505", 2024-05-15T16:05:55.212 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:55 smithi043 ceph-mon[82531]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:55.212 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:55 smithi043 ceph-mon[82531]: from='client.45006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:55.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:55 smithi043 ceph-mon[88099]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:55.212 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:55 smithi043 ceph-mon[88099]: from='client.45006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:55.238 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-05-15T16:05:55.247 INFO:tasks.cephadm:Teardown begin 2024-05-15T16:05:55.248 DEBUG:teuthology.orchestra.run.smithi043:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-15T16:05:55.277 DEBUG:teuthology.orchestra.run.smithi053:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-15T16:05:55.306 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:55 smithi053 ceph-mon[67220]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 274 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-05-15T16:05:55.307 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:55 smithi053 ceph-mon[67220]: from='client.45006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-15T16:05:55.313 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-05-15T16:05:55.313 DEBUG:teuthology.orchestra.run.smithi043:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-15T16:05:55.333 DEBUG:teuthology.orchestra.run.smithi053:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-15T16:05:55.372 INFO:tasks.cephadm:Stopping all daemons... 2024-05-15T16:05:55.372 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-05-15T16:05:55.372 DEBUG:teuthology.orchestra.run.smithi043:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.a 2024-05-15T16:05:55.585 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:55 smithi043 systemd[1]: Stopping Ceph mon.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:05:56.064 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a[82527]: 2024-05-15T16:05:55.688+0000 7f9a8bfad700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:05:56.064 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:55 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a[82527]: 2024-05-15T16:05:55.688+0000 7f9a8bfad700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-05-15T16:05:56.064 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:55 smithi043 podman[133679]: 2024-05-15 16:05:55.79217274 +0000 UTC m=+0.221454690 container died c488ab1aa87e8875e30f777ad83cba6ef4127aca8d26c407ac07409bec666cb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T16:05:56.335 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:56 smithi043 podman[133679]: 2024-05-15 16:05:56.171055209 +0000 UTC m=+0.600337157 container cleanup c488ab1aa87e8875e30f777ad83cba6ef4127aca8d26c407ac07409bec666cb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T16:05:56.336 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:56 smithi043 bash[133679]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a 2024-05-15T16:05:56.590 INFO:journalctl@ceph.mon.a.smithi043.stdout:May 15 16:05:56 smithi043 podman[133692]: 2024-05-15 16:05:56.464299686 +0000 UTC m=+0.673942744 container remove c488ab1aa87e8875e30f777ad83cba6ef4127aca8d26c407ac07409bec666cb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:05:56.590 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: ::ffff:172.21.15.53 - - [15/May/2024:16:05:56] "GET /metrics HTTP/1.1" 200 38476 "" "Prometheus/2.43.0" 2024-05-15T16:05:56.611 DEBUG:teuthology.orchestra.run.smithi043:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.a.service' 2024-05-15T16:05:56.657 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:05:56.658 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-05-15T16:05:56.658 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-05-15T16:05:56.658 DEBUG:teuthology.orchestra.run.smithi043:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.c 2024-05-15T16:05:56.969 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:56 smithi043 systemd[1]: Stopping Ceph mon.c for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:05:57.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c[88095]: 2024-05-15T16:05:56.967+0000 7fa1ef0d7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:05:57.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:56 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c[88095]: 2024-05-15T16:05:56.967+0000 7fa1ef0d7700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-05-15T16:05:57.260 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:57 smithi043 podman[133801]: 2024-05-15 16:05:57.008766317 +0000 UTC m=+0.158340323 container died d4da75e1a85c1fc123e47a40e14a21313362b4d5ebc2efd866dd52daff2c749c (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0) 2024-05-15T16:05:57.585 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:57 smithi043 podman[133801]: 2024-05-15 16:05:57.367340851 +0000 UTC m=+0.516914848 container cleanup d4da75e1a85c1fc123e47a40e14a21313362b4d5ebc2efd866dd52daff2c749c (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-05-15T16:05:57.585 INFO:journalctl@ceph.mon.c.smithi043.stdout:May 15 16:05:57 smithi043 bash[133801]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-c 2024-05-15T16:05:57.759 DEBUG:teuthology.orchestra.run.smithi043:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.c.service' 2024-05-15T16:05:57.795 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:05:57.796 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-05-15T16:05:57.796 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-05-15T16:05:57.796 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.b 2024-05-15T16:05:58.117 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:57 smithi053 systemd[1]: Stopping Ceph mon.b for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:05:58.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b[67216]: 2024-05-15T16:05:58.114+0000 7f2bc94c5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:05:58.380 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b[67216]: 2024-05-15T16:05:58.114+0000 7f2bc94c5700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-05-15T16:05:58.381 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:58 smithi053 podman[100532]: 2024-05-15 16:05:58.217464784 +0000 UTC m=+0.212465430 container died 0bcacc4488236baf14a65ab4165190a0858f3ed780781b9a96474a3da7237749 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:05:58.842 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:58 smithi053 podman[100532]: 2024-05-15 16:05:58.64862798 +0000 UTC m=+0.643628614 container cleanup 0bcacc4488236baf14a65ab4165190a0858f3ed780781b9a96474a3da7237749 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2) 2024-05-15T16:05:58.843 INFO:journalctl@ceph.mon.b.smithi053.stdout:May 15 16:05:58 smithi053 bash[100532]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mon-b 2024-05-15T16:05:59.063 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@mon.b.service' 2024-05-15T16:05:59.110 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:05:59.110 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-05-15T16:05:59.110 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-05-15T16:05:59.110 DEBUG:teuthology.orchestra.run.smithi043:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.y 2024-05-15T16:05:59.422 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:59 smithi043 systemd[1]: Stopping Ceph mgr.y for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:05:59.707 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:05:59.420+0000 7fbfc3c6f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:05:59.707 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:59 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y[62157]: 2024-05-15T16:05:59.420+0000 7fbfc3c6f700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-05-15T16:05:59.707 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:59 smithi043 podman[133922]: 2024-05-15 16:05:59.453346324 +0000 UTC m=+0.138922093 container died 8de8e1ec414e1e5d6059aa72e5778400f04f8d1d21c9f166c6badd61db01d886 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-05-15T16:06:00.015 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:59 smithi043 podman[133922]: 2024-05-15 16:05:59.813074795 +0000 UTC m=+0.498650569 container cleanup 8de8e1ec414e1e5d6059aa72e5778400f04f8d1d21c9f166c6badd61db01d886 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514) 2024-05-15T16:06:00.015 INFO:journalctl@ceph.mgr.y.smithi043.stdout:May 15 16:05:59 smithi043 bash[133922]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-y 2024-05-15T16:06:00.239 DEBUG:teuthology.orchestra.run.smithi043:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.y.service' 2024-05-15T16:06:00.276 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:06:00.276 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-05-15T16:06:00.276 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-05-15T16:06:00.277 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.x 2024-05-15T16:06:00.618 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 16:06:00 smithi053 systemd[1]: Stopping Ceph mgr.x for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:06:00.879 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 16:06:00 smithi053 podman[100654]: 2024-05-15 16:06:00.617601176 +0000 UTC m=+0.133088852 container died 006c00101d675e8c77d581e5086ba26be7c298eff75f065910680f593119c9ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d) 2024-05-15T16:06:01.330 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 16:06:01 smithi053 podman[100654]: 2024-05-15 16:06:01.136665698 +0000 UTC m=+0.652153374 container cleanup 006c00101d675e8c77d581e5086ba26be7c298eff75f065910680f593119c9ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-05-15T16:06:01.331 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 16:06:01 smithi053 bash[100654]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x 2024-05-15T16:06:01.581 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 16:06:01 smithi053 podman[100667]: 2024-05-15 16:06:01.32975391 +0000 UTC m=+0.712469779 container remove 006c00101d675e8c77d581e5086ba26be7c298eff75f065910680f593119c9ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-mgr-x, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-05-15T16:06:01.582 INFO:journalctl@ceph.mgr.x.smithi053.stdout:May 15 16:06:01 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-05-15T16:06:01.586 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@mgr.x.service' 2024-05-15T16:06:01.622 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:06:01.622 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-05-15T16:06:01.622 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-05-15T16:06:01.623 DEBUG:teuthology.orchestra.run.smithi043:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.0 2024-05-15T16:06:01.927 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:01 smithi043 systemd[1]: Stopping Ceph osd.0 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:06:02.335 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[95970]: 2024-05-15T16:06:01.925+0000 7f3cddf42700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:06:02.335 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[95970]: 2024-05-15T16:06:01.925+0000 7f3cddf42700 -1 osd.0 160 *** Got signal Terminated *** 2024-05-15T16:06:02.336 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:01 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0[95970]: 2024-05-15T16:06:01.925+0000 7f3cddf42700 -1 osd.0 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:06:03.835 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:03.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T16:06:03.835 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:03.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T16:06:07.705 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:07 smithi043 podman[134042]: 2024-05-15 16:06:07.428328499 +0000 UTC m=+5.607340416 container died d646f037d33caa495ddcf065774f1d2982f990746d26bf9dae252f87b31889e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-05-15T16:06:08.085 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:07 smithi043 podman[134042]: 2024-05-15 16:06:07.877540727 +0000 UTC m=+6.056552682 container cleanup d646f037d33caa495ddcf065774f1d2982f990746d26bf9dae252f87b31889e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-05-15T16:06:08.086 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:07 smithi043 bash[134042]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0 2024-05-15T16:06:08.530 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:08 smithi043 podman[134060]: 2024-05-15 16:06:08.087276285 +0000 UTC m=+0.650832897 container remove d646f037d33caa495ddcf065774f1d2982f990746d26bf9dae252f87b31889e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-05-15T16:06:08.833 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:08 smithi043 podman[134146]: 2024-05-15 16:06:08.529836392 +0000 UTC m=+0.114503978 container create 2dec9390cafe5a4edb3a7acce54f2da9cdda7d39cf4c9716a5081d72a4e2dd91 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:06:08.833 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:08 smithi043 podman[134146]: 2024-05-15 16:06:08.433221357 +0000 UTC m=+0.017888954 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:06:08.833 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:08 smithi043 podman[134146]: 2024-05-15 16:06:08.740786683 +0000 UTC m=+0.325454272 container init 2dec9390cafe5a4edb3a7acce54f2da9cdda7d39cf4c9716a5081d72a4e2dd91 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-15T16:06:08.833 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:08 smithi043 podman[134146]: 2024-05-15 16:06:08.744211439 +0000 UTC m=+0.328879034 container start 2dec9390cafe5a4edb3a7acce54f2da9cdda7d39cf4c9716a5081d72a4e2dd91 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-05-15T16:06:08.833 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:08 smithi043 podman[134146]: 2024-05-15 16:06:08.832384791 +0000 UTC m=+0.417052390 container attach 2dec9390cafe5a4edb3a7acce54f2da9cdda7d39cf4c9716a5081d72a4e2dd91 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-05-15T16:06:09.085 INFO:journalctl@ceph.osd.0.smithi043.stdout:May 15 16:06:09 smithi043 podman[134146]: 2024-05-15 16:06:09.055372888 +0000 UTC m=+0.640040483 container died 2dec9390cafe5a4edb3a7acce54f2da9cdda7d39cf4c9716a5081d72a4e2dd91 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-0-deactivate, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:06:09.745 DEBUG:teuthology.orchestra.run.smithi043:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.0.service' 2024-05-15T16:06:09.789 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:06:09.789 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-05-15T16:06:09.789 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-05-15T16:06:09.790 DEBUG:teuthology.orchestra.run.smithi043:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.1 2024-05-15T16:06:10.167 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:09 smithi043 systemd[1]: Stopping Ceph osd.1 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:06:10.585 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[100795]: 2024-05-15T16:06:10.164+0000 7f02020d0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:06:10.585 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[100795]: 2024-05-15T16:06:10.164+0000 7f02020d0700 -1 osd.1 160 *** Got signal Terminated *** 2024-05-15T16:06:10.585 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1[100795]: 2024-05-15T16:06:10.164+0000 7f02020d0700 -1 osd.1 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:06:13.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:13.551Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T16:06:13.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:13.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T16:06:13.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:13 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:13.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T16:06:15.758 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:15 smithi043 podman[134265]: 2024-05-15 16:06:15.480920218 +0000 UTC m=+5.439554358 container died 9bdd94dfda49a86787d198dec23f2c86c7f23ed00bbf5c1f150b9b82c4c917ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T16:06:16.085 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:15 smithi043 podman[134265]: 2024-05-15 16:06:15.847879508 +0000 UTC m=+5.806513645 container cleanup 9bdd94dfda49a86787d198dec23f2c86c7f23ed00bbf5c1f150b9b82c4c917ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-05-15T16:06:16.085 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:15 smithi043 bash[134265]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1 2024-05-15T16:06:16.484 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:16 smithi043 podman[134283]: 2024-05-15 16:06:16.140740439 +0000 UTC m=+0.653418966 container remove 9bdd94dfda49a86787d198dec23f2c86c7f23ed00bbf5c1f150b9b82c4c917ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-05-15T16:06:16.835 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:16 smithi043 podman[134375]: 2024-05-15 16:06:16.483734957 +0000 UTC m=+0.105566850 container create 3305a01aba55a5ac4e6e9a1bf8dc7fee77845421f5a0e0bfffe9c2a5e812a0de (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-05-15T16:06:16.835 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:16 smithi043 podman[134375]: 2024-05-15 16:06:16.396194461 +0000 UTC m=+0.018026372 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:06:16.836 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:16 smithi043 podman[134375]: 2024-05-15 16:06:16.669809873 +0000 UTC m=+0.291641776 container init 3305a01aba55a5ac4e6e9a1bf8dc7fee77845421f5a0e0bfffe9c2a5e812a0de (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-05-15T16:06:16.836 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:16 smithi043 podman[134375]: 2024-05-15 16:06:16.673215686 +0000 UTC m=+0.295047578 container start 3305a01aba55a5ac4e6e9a1bf8dc7fee77845421f5a0e0bfffe9c2a5e812a0de (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:06:16.836 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:16 smithi043 podman[134375]: 2024-05-15 16:06:16.727869741 +0000 UTC m=+0.349701636 container attach 3305a01aba55a5ac4e6e9a1bf8dc7fee77845421f5a0e0bfffe9c2a5e812a0de (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-05-15T16:06:17.248 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:17 smithi043 podman[134402]: 2024-05-15 16:06:17.007129114 +0000 UTC m=+0.018872457 container died 3305a01aba55a5ac4e6e9a1bf8dc7fee77845421f5a0e0bfffe9c2a5e812a0de (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240514, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-05-15T16:06:17.499 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:17 smithi043 podman[134402]: 2024-05-15 16:06:17.481312173 +0000 UTC m=+0.493055507 container remove 3305a01aba55a5ac4e6e9a1bf8dc7fee77845421f5a0e0bfffe9c2a5e812a0de (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-1-deactivate, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:06:17.499 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:17 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.1.service: Deactivated successfully. 2024-05-15T16:06:17.499 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:17 smithi043 systemd[1]: Stopped Ceph osd.1 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:06:17.499 INFO:journalctl@ceph.osd.1.smithi043.stdout:May 15 16:06:17 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.1.service: Consumed 6.906s CPU time. 2024-05-15T16:06:17.502 DEBUG:teuthology.orchestra.run.smithi043:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.1.service' 2024-05-15T16:06:17.545 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:06:17.545 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-05-15T16:06:17.545 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-05-15T16:06:17.545 DEBUG:teuthology.orchestra.run.smithi043:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.2 2024-05-15T16:06:17.835 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:17 smithi043 systemd[1]: Stopping Ceph osd.2 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:06:18.335 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[107990]: 2024-05-15T16:06:17.883+0000 7f686f6db700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:06:18.335 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[107990]: 2024-05-15T16:06:17.883+0000 7f686f6db700 -1 osd.2 160 *** Got signal Terminated *** 2024-05-15T16:06:18.335 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:17 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2[107990]: 2024-05-15T16:06:17.883+0000 7f686f6db700 -1 osd.2 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:06:23.711 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:23 smithi043 podman[134483]: 2024-05-15 16:06:23.393241136 +0000 UTC m=+5.612996671 container died 087a14715d561ee617c3aab791bdfd792560e003b403b554080a2245197cab94 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-15T16:06:23.712 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:23.552Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T16:06:23.712 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:23.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T16:06:23.712 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:23 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:23.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T16:06:24.053 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:23 smithi043 podman[134483]: 2024-05-15 16:06:23.817705163 +0000 UTC m=+6.037460688 container cleanup 087a14715d561ee617c3aab791bdfd792560e003b403b554080a2245197cab94 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:06:24.053 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:23 smithi043 bash[134483]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2 2024-05-15T16:06:24.335 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:24 smithi043 podman[134500]: 2024-05-15 16:06:24.05261117 +0000 UTC m=+0.659079706 container remove 087a14715d561ee617c3aab791bdfd792560e003b403b554080a2245197cab94 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2) 2024-05-15T16:06:24.832 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:24 smithi043 podman[134597]: 2024-05-15 16:06:24.437554607 +0000 UTC m=+0.018681358 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:06:24.832 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:24 smithi043 podman[134597]: 2024-05-15 16:06:24.545180962 +0000 UTC m=+0.126307702 container create 320f85fb08f70f7a0a77ab1997dfb8d93876a2cf1efdd2115effda8b4761909e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-05-15T16:06:24.832 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:24 smithi043 podman[134597]: 2024-05-15 16:06:24.731091956 +0000 UTC m=+0.312218696 container init 320f85fb08f70f7a0a77ab1997dfb8d93876a2cf1efdd2115effda8b4761909e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-05-15T16:06:24.832 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:24 smithi043 podman[134597]: 2024-05-15 16:06:24.734685325 +0000 UTC m=+0.315812074 container start 320f85fb08f70f7a0a77ab1997dfb8d93876a2cf1efdd2115effda8b4761909e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d) 2024-05-15T16:06:24.832 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:24 smithi043 podman[134597]: 2024-05-15 16:06:24.831015531 +0000 UTC m=+0.412142288 container attach 320f85fb08f70f7a0a77ab1997dfb8d93876a2cf1efdd2115effda8b4761909e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:06:25.085 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:25 smithi043 podman[134597]: 2024-05-15 16:06:25.053857121 +0000 UTC m=+0.634983860 container died 320f85fb08f70f7a0a77ab1997dfb8d93876a2cf1efdd2115effda8b4761909e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d) 2024-05-15T16:06:25.750 DEBUG:teuthology.orchestra.run.smithi043:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.2.service' 2024-05-15T16:06:25.777 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:25 smithi043 podman[134624]: 2024-05-15 16:06:25.726234049 +0000 UTC m=+0.665426361 container remove 320f85fb08f70f7a0a77ab1997dfb8d93876a2cf1efdd2115effda8b4761909e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD) 2024-05-15T16:06:25.778 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:25 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.2.service: Deactivated successfully. 2024-05-15T16:06:25.778 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:25 smithi043 systemd[1]: Stopped Ceph osd.2 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:06:25.778 INFO:journalctl@ceph.osd.2.smithi043.stdout:May 15 16:06:25 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.2.service: Consumed 5.415s CPU time. 2024-05-15T16:06:25.796 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:06:25.796 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-05-15T16:06:25.796 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-05-15T16:06:25.797 DEBUG:teuthology.orchestra.run.smithi043:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.3 2024-05-15T16:06:26.086 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:25 smithi043 systemd[1]: Stopping Ceph osd.3 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:06:26.585 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[114748]: 2024-05-15T16:06:26.145+0000 7f77040a9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:06:26.585 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[114748]: 2024-05-15T16:06:26.145+0000 7f77040a9700 -1 osd.3 160 *** Got signal Terminated *** 2024-05-15T16:06:26.586 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:26 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3[114748]: 2024-05-15T16:06:26.145+0000 7f77040a9700 -1 osd.3 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:06:29.003 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:28 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:28.699+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:29.264 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:29.002+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:29.629 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:29.463+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:29.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:29.375+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:30.023 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:29 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:29.710+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:30.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:30.022+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:30.756 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:30.479+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:30.756 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:30.392+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:31.036 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:30 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:30.754+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:31.357 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:31.034+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:31.629 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:31.526+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:31.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:31.355+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:31.835 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:31 smithi043 podman[134704]: 2024-05-15 16:06:31.487095464 +0000 UTC m=+5.456207887 container died 258344f90360e1e5e457c9699138d6ee8e5a6457b470714fb20d68b3e536d6d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-15T16:06:32.105 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:31 smithi043 podman[134704]: 2024-05-15 16:06:31.911833182 +0000 UTC m=+5.880945612 container cleanup 258344f90360e1e5e457c9699138d6ee8e5a6457b470714fb20d68b3e536d6d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:06:32.106 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:31 smithi043 bash[134704]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3 2024-05-15T16:06:32.129 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:31 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:31.791+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:32.130 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:32 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:32.030+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:32.540 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:32 smithi043 podman[134722]: 2024-05-15 16:06:32.104819575 +0000 UTC m=+0.615342511 container remove 258344f90360e1e5e457c9699138d6ee8e5a6457b470714fb20d68b3e536d6d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-05-15T16:06:32.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:32 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:32.363+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:32.630 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:32 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:32.533+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:32.809 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:32 smithi043 podman[134811]: 2024-05-15 16:06:32.539080717 +0000 UTC m=+0.104027299 container create c1893c102cac48616f5f181a05acc500e37ad83530b05f8b378fce65181aac99 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-05-15T16:06:32.810 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:32 smithi043 podman[134811]: 2024-05-15 16:06:32.453213294 +0000 UTC m=+0.018159854 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:06:32.810 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:32 smithi043 podman[134811]: 2024-05-15 16:06:32.750506297 +0000 UTC m=+0.315452848 container init c1893c102cac48616f5f181a05acc500e37ad83530b05f8b378fce65181aac99 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-05-15T16:06:32.810 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:32 smithi043 podman[134811]: 2024-05-15 16:06:32.754134548 +0000 UTC m=+0.319081088 container start c1893c102cac48616f5f181a05acc500e37ad83530b05f8b378fce65181aac99 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T16:06:32.810 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:32 smithi043 podman[134811]: 2024-05-15 16:06:32.808518507 +0000 UTC m=+0.373465056 container attach c1893c102cac48616f5f181a05acc500e37ad83530b05f8b378fce65181aac99 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-05-15T16:06:33.075 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:32 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:32.746+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:33.332 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:33.073+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:33.335 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:33 smithi043 podman[134811]: 2024-05-15 16:06:33.065305868 +0000 UTC m=+0.630252417 container died c1893c102cac48616f5f181a05acc500e37ad83530b05f8b378fce65181aac99 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T16:06:33.567 DEBUG:teuthology.orchestra.run.smithi043:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.3.service' 2024-05-15T16:06:33.591 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:33 smithi043 podman[134837]: 2024-05-15 16:06:33.545156991 +0000 UTC m=+0.470471951 container remove c1893c102cac48616f5f181a05acc500e37ad83530b05f8b378fce65181aac99 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-15T16:06:33.591 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:33 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.3.service: Deactivated successfully. 2024-05-15T16:06:33.591 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:33 smithi043 systemd[1]: Stopped Ceph osd.3 for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:06:33.591 INFO:journalctl@ceph.osd.3.smithi043.stdout:May 15 16:06:33 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.3.service: Consumed 6.873s CPU time. 2024-05-15T16:06:33.592 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:33.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T16:06:33.592 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:33.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T16:06:33.592 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:33 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:33.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T16:06:33.608 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:06:33.608 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-05-15T16:06:33.608 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-05-15T16:06:33.609 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.4 2024-05-15T16:06:33.629 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:33.330+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:33.629 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:33.525+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:33.924 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:33 smithi053 systemd[1]: Stopping Ceph osd.4 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:06:33.925 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:33.772+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:33.925 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:33.922+0000 7f881f919700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:06:33.925 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:33.922+0000 7f881f919700 -1 osd.4 160 *** Got signal Terminated *** 2024-05-15T16:06:34.325 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:33 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:33.922+0000 7f881f919700 -1 osd.4 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:06:34.325 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:34.069+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:34.629 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:34.527+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:34.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:34.323+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:35.107 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:34 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:34.759+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:35.359 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:35.105+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:35.630 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:35.488+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:35.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:35.358+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:36.129 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:35 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:35.759+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:36.508 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:36.324+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:36.508 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:36.324+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:36.508 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:36.138+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:36.508 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:36.138+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:36.879 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:36.715+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:36.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:36 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:36.506+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:37.514 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:37.330+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:37.515 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:37.330+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:37.515 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:37.129+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:37.515 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:37.129+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:37.879 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:37.755+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:37.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:37.513+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:38.539 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:38.326+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:38.539 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:38.326+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:38.540 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:38.167+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:38.540 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:38.167+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:38.879 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4[77724]: 2024-05-15T16:06:38.758+0000 7f8817f05700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:04.156114+0000 front 2024-05-15T16:06:04.156253+0000 (oldest deadline 2024-05-15T16:06:28.255994+0000) 2024-05-15T16:06:38.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:38.538+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:38.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:06:38.664Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.43:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.43:8765: connect: connection refused" 2024-05-15T16:06:38.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:06:38.665Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.43:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.43:8765: connect: connection refused" 2024-05-15T16:06:38.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:06:38.665Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.43:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.43:8765: connect: connection refused" 2024-05-15T16:06:38.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:06:38.666Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.43:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.43:8765: connect: connection refused" 2024-05-15T16:06:38.880 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:06:38 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:06:38.666Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.43:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.43:8765: connect: connection refused" 2024-05-15T16:06:39.391 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:39 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:39.295+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:39.455 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:39 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:39.295+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:39.456 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:39 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:39.138+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:39.456 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:39 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:39.138+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:39.699 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:39 smithi053 podman[100789]: 2024-05-15 16:06:39.390376801 +0000 UTC m=+5.573343702 container died b342d0fab575f9eb661f0fc0c70891f44acec27de13b902837c1be1dbf8b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-05-15T16:06:39.699 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:39 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:39.570+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:39.699 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:39 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:39.570+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:14.441166+0000 front 2024-05-15T16:06:14.441251+0000 (oldest deadline 2024-05-15T16:06:38.541071+0000) 2024-05-15T16:06:40.278 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:39 smithi053 podman[100789]: 2024-05-15 16:06:39.964325484 +0000 UTC m=+6.147292390 container cleanup b342d0fab575f9eb661f0fc0c70891f44acec27de13b902837c1be1dbf8b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-05-15T16:06:40.278 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:39 smithi053 bash[100789]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4 2024-05-15T16:06:40.279 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:40.186+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:40.279 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:40.186+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:40.596 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:40 smithi053 podman[100807]: 2024-05-15 16:06:40.316384445 +0000 UTC m=+0.926244839 container remove b342d0fab575f9eb661f0fc0c70891f44acec27de13b902837c1be1dbf8b1efc (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-05-15T16:06:40.596 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:40.295+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:40.597 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:40.295+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:40.879 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:40.595+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:40.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:40 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:40.595+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:14.441166+0000 front 2024-05-15T16:06:14.441251+0000 (oldest deadline 2024-05-15T16:06:38.541071+0000) 2024-05-15T16:06:40.880 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:40 smithi053 podman[100902]: 2024-05-15 16:06:40.562352878 +0000 UTC m=+0.018135175 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:06:40.880 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:40 smithi053 podman[100902]: 2024-05-15 16:06:40.684424347 +0000 UTC m=+0.140206679 container create a5897b16c9c6b013f7b1685379782ba23e88dc04ea98c55cceb581a48fc41ca1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True) 2024-05-15T16:06:41.231 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:41 smithi053 podman[100902]: 2024-05-15 16:06:41.012280149 +0000 UTC m=+0.468062439 container init a5897b16c9c6b013f7b1685379782ba23e88dc04ea98c55cceb581a48fc41ca1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-15T16:06:41.231 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:41 smithi053 podman[100902]: 2024-05-15 16:06:41.016180426 +0000 UTC m=+0.471962716 container start a5897b16c9c6b013f7b1685379782ba23e88dc04ea98c55cceb581a48fc41ca1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d) 2024-05-15T16:06:41.232 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:41 smithi053 podman[100902]: 2024-05-15 16:06:41.086754471 +0000 UTC m=+0.542536761 container attach a5897b16c9c6b013f7b1685379782ba23e88dc04ea98c55cceb581a48fc41ca1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-15T16:06:41.568 INFO:journalctl@ceph.osd.4.smithi053.stdout:May 15 16:06:41 smithi053 podman[100902]: 2024-05-15 16:06:41.321487275 +0000 UTC m=+0.777269565 container died a5897b16c9c6b013f7b1685379782ba23e88dc04ea98c55cceb581a48fc41ca1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-05-15T16:06:41.568 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:41 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:41.342+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:41.568 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:41 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:41.342+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:41.568 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:41 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:41.230+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:41.569 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:41 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:41.230+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:41.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:41 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:41.566+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:41.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:41 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:41.566+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:14.441166+0000 front 2024-05-15T16:06:14.441251+0000 (oldest deadline 2024-05-15T16:06:38.541071+0000) 2024-05-15T16:06:41.926 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.4.service' 2024-05-15T16:06:42.009 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:06:42.009 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-05-15T16:06:42.009 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-05-15T16:06:42.009 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.5 2024-05-15T16:06:42.202 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:42 smithi053 systemd[1]: Stopping Ceph osd.5 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:06:42.552 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:42.315+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:42.552 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:42.315+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:42.552 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:42.200+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:42.552 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:42.200+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:42.553 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:42.393+0000 7f76967b4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:06:42.553 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:42.393+0000 7f76967b4700 -1 osd.5 160 *** Got signal Terminated *** 2024-05-15T16:06:42.553 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:42.393+0000 7f76967b4700 -1 osd.5 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:06:42.879 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:42.559+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:42.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:42 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:42.559+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:14.441166+0000 front 2024-05-15T16:06:14.441251+0000 (oldest deadline 2024-05-15T16:06:38.541071+0000) 2024-05-15T16:06:43.582 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:43.300+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:43.582 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:43.300+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:43.583 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:43.165+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:43.583 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:43.165+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:43.835 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:43.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T16:06:43.835 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:43.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T16:06:43.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:43 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:43.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T16:06:43.879 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:43.581+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:43.879 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:43 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:43.581+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:14.441166+0000 front 2024-05-15T16:06:14.441251+0000 (oldest deadline 2024-05-15T16:06:38.541071+0000) 2024-05-15T16:06:44.554 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:44.290+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:44.554 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:44.290+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:44.554 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:44.290+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:44.554 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:44.165+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:44.555 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:44.166+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:44.879 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:44.552+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:44.879 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:44 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:44.552+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:14.441166+0000 front 2024-05-15T16:06:14.441251+0000 (oldest deadline 2024-05-15T16:06:38.541071+0000) 2024-05-15T16:06:45.560 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:45.173+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:45.560 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:45.173+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:45.561 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:45.311+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:45.561 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:45.311+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:45.561 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:45.311+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:45.879 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:45.558+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:45.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:45.558+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:14.441166+0000 front 2024-05-15T16:06:14.441251+0000 (oldest deadline 2024-05-15T16:06:38.541071+0000) 2024-05-15T16:06:45.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:45 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:45.558+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:20.842305+0000 front 2024-05-15T16:06:20.842154+0000 (oldest deadline 2024-05-15T16:06:45.542219+0000) 2024-05-15T16:06:46.602 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:46.282+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:46.602 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:46.282+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:46.602 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:46.282+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:46.602 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:46.180+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:46.602 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:46.180+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:46.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:46.600+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:01.038885+0000 front 2024-05-15T16:06:01.038924+0000 (oldest deadline 2024-05-15T16:06:28.639498+0000) 2024-05-15T16:06:46.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:46.600+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:14.441166+0000 front 2024-05-15T16:06:14.441251+0000 (oldest deadline 2024-05-15T16:06:38.541071+0000) 2024-05-15T16:06:46.880 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:46 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5[82228]: 2024-05-15T16:06:46.600+0000 7f768eda0700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:20.842305+0000 front 2024-05-15T16:06:20.842154+0000 (oldest deadline 2024-05-15T16:06:45.542219+0000) 2024-05-15T16:06:47.629 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:47.154+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:47.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:47.154+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:47.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:47.154+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:47.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:47.324+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:47.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:47.324+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:47.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:47 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:47.324+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:48.117 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:47 smithi053 podman[101019]: 2024-05-15 16:06:47.788284323 +0000 UTC m=+5.546818367 container died f709584ed028a4fe85909b3c8253c158dd3ba0ab75652df13c215339b921f79e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T16:06:48.379 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:48 smithi053 podman[101019]: 2024-05-15 16:06:48.194264049 +0000 UTC m=+5.952798092 container cleanup f709584ed028a4fe85909b3c8253c158dd3ba0ab75652df13c215339b921f79e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-05-15T16:06:48.380 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:48 smithi053 bash[101019]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5 2024-05-15T16:06:48.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:48.116+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:48.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:48.116+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:48.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:48.116+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:48.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:48.295+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:48.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:48.295+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:48.381 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:48 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:48.295+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:48.755 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:48 smithi053 podman[101035]: 2024-05-15 16:06:48.420944371 +0000 UTC m=+0.630619207 container remove f709584ed028a4fe85909b3c8253c158dd3ba0ab75652df13c215339b921f79e (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-05-15T16:06:49.025 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:48 smithi053 podman[101124]: 2024-05-15 16:06:48.657532442 +0000 UTC m=+0.017688819 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:06:49.025 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:48 smithi053 podman[101124]: 2024-05-15 16:06:48.755105629 +0000 UTC m=+0.115262051 container create ca3ffb230c01bc25ef04f589a64f09714340402194a18733351e87f6cf8a6921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-05-15T16:06:49.025 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:48 smithi053 podman[101124]: 2024-05-15 16:06:48.949436038 +0000 UTC m=+0.309592409 container init ca3ffb230c01bc25ef04f589a64f09714340402194a18733351e87f6cf8a6921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T16:06:49.025 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:48 smithi053 podman[101124]: 2024-05-15 16:06:48.952822608 +0000 UTC m=+0.312978979 container start ca3ffb230c01bc25ef04f589a64f09714340402194a18733351e87f6cf8a6921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-05-15T16:06:49.025 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:49 smithi053 podman[101124]: 2024-05-15 16:06:49.024058295 +0000 UTC m=+0.384214682 container attach ca3ffb230c01bc25ef04f589a64f09714340402194a18733351e87f6cf8a6921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:06:49.289 INFO:journalctl@ceph.osd.5.smithi053.stdout:May 15 16:06:49 smithi053 podman[101124]: 2024-05-15 16:06:49.253419197 +0000 UTC m=+0.613575569 container died ca3ffb230c01bc25ef04f589a64f09714340402194a18733351e87f6cf8a6921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, RELEASE=wip-65480-squid-de5210d, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:06:49.289 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:49.108+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:49.290 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:49.108+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:49.290 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:49.108+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:49.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:49.287+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:49.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:49.287+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:49.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:49 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:49.287+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:49.840 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.5.service' 2024-05-15T16:06:49.882 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:06:49.883 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-05-15T16:06:49.883 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-05-15T16:06:49.883 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.6 2024-05-15T16:06:50.115 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:49 smithi053 systemd[1]: Stopping Ceph osd.6 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:06:50.379 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:50.114+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:50.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:50.114+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:50.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:50.114+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:50.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:50.230+0000 7fb531681700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:06:50.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:50.230+0000 7fb531681700 -1 osd.6 160 *** Got signal Terminated *** 2024-05-15T16:06:50.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:50.230+0000 7fb531681700 -1 osd.6 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:06:50.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:50.311+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:50.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:50.311+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:50.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:50 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:50.312+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:51.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:51 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:51.095+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:51.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:51 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:51.095+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:51.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:51 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:51.095+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:51.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:51 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:51.323+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:51.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:51 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:51.323+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:51.380 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:51 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:51.323+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:52.327 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:52.134+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:52.327 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:52.134+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:52.327 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:52.134+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:52.327 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:52.134+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:06:52.628 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:52.325+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:52.628 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:52.325+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:52.628 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:52 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:52.325+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:53.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:53.374+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:53.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:53.374+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:53.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:53.374+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:53.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:53.374+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.464816+0000 front 2024-05-15T16:06:26.464827+0000 (oldest deadline 2024-05-15T16:06:52.865108+0000) 2024-05-15T16:06:53.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:53.168+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:53.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:53.168+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:53.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:53.168+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:53.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:53 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:53.168+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:06:53.835 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:53.555Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T16:06:53.835 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:53.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T16:06:53.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:06:53 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:06:53.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T16:06:54.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:54.328+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:05.060588+0000 front 2024-05-15T16:06:05.060618+0000 (oldest deadline 2024-05-15T16:06:29.160837+0000) 2024-05-15T16:06:54.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:54.328+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:13.762340+0000 front 2024-05-15T16:06:13.762330+0000 (oldest deadline 2024-05-15T16:06:36.062264+0000) 2024-05-15T16:06:54.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:54.328+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.463816+0000 front 2024-05-15T16:06:22.463827+0000 (oldest deadline 2024-05-15T16:06:43.563735+0000) 2024-05-15T16:06:54.630 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6[87029]: 2024-05-15T16:06:54.328+0000 7fb52dc75700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.464816+0000 front 2024-05-15T16:06:26.464827+0000 (oldest deadline 2024-05-15T16:06:52.865108+0000) 2024-05-15T16:06:54.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:54.133+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:54.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:54.133+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:54.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:54.133+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:54.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:54 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:54.133+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:06:55.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:55.164+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:55.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:55.164+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:55.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:55.164+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:55.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:55 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:55.164+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:06:55.899 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:55 smithi053 podman[101229]: 2024-05-15 16:06:55.642256914 +0000 UTC m=+5.520114243 container died d5353ef723114a7838c03c46283d6f5fa8c2d8c4c7ae5ee0024d3ad3f3e82664 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514) 2024-05-15T16:06:56.169 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:55 smithi053 podman[101229]: 2024-05-15 16:06:55.988846676 +0000 UTC m=+5.866703991 container cleanup d5353ef723114a7838c03c46283d6f5fa8c2d8c4c7ae5ee0024d3ad3f3e82664 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:06:56.169 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:55 smithi053 bash[101229]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6 2024-05-15T16:06:56.625 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:56 smithi053 podman[101249]: 2024-05-15 16:06:56.173418221 +0000 UTC m=+0.522430408 container remove d5353ef723114a7838c03c46283d6f5fa8c2d8c4c7ae5ee0024d3ad3f3e82664 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-65480-squid-de5210d, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True) 2024-05-15T16:06:56.626 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:56.167+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:56.626 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:56.167+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:56.626 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:56.167+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:56.626 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:56 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:56.167+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:06:56.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:56 smithi053 podman[101341]: 2024-05-15 16:06:56.624735762 +0000 UTC m=+0.104287589 container create 4ee9214a257993ae1482d31fe16d31db928a1e52e2428d92bb9f8368dc97f62c (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, io.buildah.version=1.33.5) 2024-05-15T16:06:56.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:56 smithi053 podman[101341]: 2024-05-15 16:06:56.538985442 +0000 UTC m=+0.018537271 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:06:56.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:56 smithi053 podman[101341]: 2024-05-15 16:06:56.827064357 +0000 UTC m=+0.306616177 container init 4ee9214a257993ae1482d31fe16d31db928a1e52e2428d92bb9f8368dc97f62c (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:06:56.880 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:56 smithi053 podman[101341]: 2024-05-15 16:06:56.830326841 +0000 UTC m=+0.309878646 container start 4ee9214a257993ae1482d31fe16d31db928a1e52e2428d92bb9f8368dc97f62c (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, ceph=True) 2024-05-15T16:06:57.207 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:56 smithi053 podman[101341]: 2024-05-15 16:06:56.920347609 +0000 UTC m=+0.399899430 container attach 4ee9214a257993ae1482d31fe16d31db928a1e52e2428d92bb9f8368dc97f62c (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_BRANCH=HEAD) 2024-05-15T16:06:57.208 INFO:journalctl@ceph.osd.6.smithi053.stdout:May 15 16:06:57 smithi053 podman[101341]: 2024-05-15 16:06:57.133447653 +0000 UTC m=+0.612999478 container died 4ee9214a257993ae1482d31fe16d31db928a1e52e2428d92bb9f8368dc97f62c (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-05-15T16:06:57.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:57 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:57.206+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:57.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:57 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:57.206+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:57.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:57 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:57.206+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:57.630 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:57 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:57.206+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:06:57.690 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.6.service' 2024-05-15T16:06:57.735 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:06:57.735 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-05-15T16:06:57.735 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-05-15T16:06:57.735 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.7 2024-05-15T16:06:58.067 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:57 smithi053 systemd[1]: Stopping Ceph osd.7 for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:06:58.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:58.066+0000 7fd43be3e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-15T16:06:58.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:58.066+0000 7fd43be3e700 -1 osd.7 160 *** Got signal Terminated *** 2024-05-15T16:06:58.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:58.066+0000 7fd43be3e700 -1 osd.7 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-15T16:06:58.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:58.180+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:58.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:58.180+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:58.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:58.180+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:58.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:58 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:58.180+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:06:59.545 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:59.132+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:06:59.546 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:59.132+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:06:59.546 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:59.132+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:06:59.546 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:06:59 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:06:59.132+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:07:00.379 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:00.088+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:07:00.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:00.088+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:07:00.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:00.088+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:07:00.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:00 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:00.088+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:07:01.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:01.077+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:07:01.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:01.077+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:07:01.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:01.077+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:07:01.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:01 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:01.077+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:07:02.379 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:02 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:02.050+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:07:02.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:02 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:02.050+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:07:02.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:02 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:02.050+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:07:02.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:02 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:02.050+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:07:02.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:02 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:02.050+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.53:6804 osd.4 since back 2024-05-15T16:06:37.992153+0000 front 2024-05-15T16:06:37.992250+0000 (oldest deadline 2024-05-15T16:07:01.492083+0000) 2024-05-15T16:07:03.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:03.009+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6806 osd.0 since back 2024-05-15T16:06:03.587074+0000 front 2024-05-15T16:06:03.587212+0000 (oldest deadline 2024-05-15T16:06:28.886828+0000) 2024-05-15T16:07:03.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:03.010+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6814 osd.1 since back 2024-05-15T16:06:12.988108+0000 front 2024-05-15T16:06:12.988040+0000 (oldest deadline 2024-05-15T16:06:35.887950+0000) 2024-05-15T16:07:03.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:03.010+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6822 osd.2 since back 2024-05-15T16:06:22.190265+0000 front 2024-05-15T16:06:22.190234+0000 (oldest deadline 2024-05-15T16:06:46.290095+0000) 2024-05-15T16:07:03.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:03.010+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.43:6830 osd.3 since back 2024-05-15T16:06:26.290657+0000 front 2024-05-15T16:06:26.290750+0000 (oldest deadline 2024-05-15T16:06:51.590679+0000) 2024-05-15T16:07:03.380 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:03 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7[91970]: 2024-05-15T16:07:03.010+0000 7fd43442a700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.53:6804 osd.4 since back 2024-05-15T16:06:37.992153+0000 front 2024-05-15T16:06:37.992250+0000 (oldest deadline 2024-05-15T16:07:01.492083+0000) 2024-05-15T16:07:03.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:07:03.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T16:07:03.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:07:03.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi043.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.43:8443: connect: connection refused" 2024-05-15T16:07:03.836 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:03 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:07:03.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi053.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.53:8443: connect: connection refused" 2024-05-15T16:07:03.880 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:03 smithi053 podman[101444]: 2024-05-15 16:07:03.555616319 +0000 UTC m=+5.582383274 container died 784f18f73ebde8c67b43763c8aed670eea1f4c3dcba305550ba4cfc66e3dc9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514) 2024-05-15T16:07:04.269 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:03 smithi053 podman[101444]: 2024-05-15 16:07:03.95011355 +0000 UTC m=+5.976880526 container cleanup 784f18f73ebde8c67b43763c8aed670eea1f4c3dcba305550ba4cfc66e3dc9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-65480-squid-de5210d, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-15T16:07:04.269 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:03 smithi053 bash[101444]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7 2024-05-15T16:07:04.629 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:04 smithi053 podman[101474]: 2024-05-15 16:07:04.26850122 +0000 UTC m=+0.708278709 container remove 784f18f73ebde8c67b43763c8aed670eea1f4c3dcba305550ba4cfc66e3dc9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-65480-squid-de5210d, ceph=True, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-05-15T16:07:04.972 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:04 smithi053 podman[101566]: 2024-05-15 16:07:04.547131183 +0000 UTC m=+0.018610405 image pull 48bc18eb22d8284fb8ad68bf46e17b174a8aaf15dd2581bf7f093c432d12db31 quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a 2024-05-15T16:07:04.972 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:04 smithi053 podman[101566]: 2024-05-15 16:07:04.652716659 +0000 UTC m=+0.124195910 container create 2ebe15e0c767920c6ad4bffd39218d0ce79869ce8d55bc5d9c65e0e34cbf47af (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-65480-squid-de5210d, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-05-15T16:07:04.972 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:04 smithi053 podman[101566]: 2024-05-15 16:07:04.888359508 +0000 UTC m=+0.359838717 container init 2ebe15e0c767920c6ad4bffd39218d0ce79869ce8d55bc5d9c65e0e34cbf47af (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-15T16:07:04.972 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:04 smithi053 podman[101566]: 2024-05-15 16:07:04.891585186 +0000 UTC m=+0.363064396 container start 2ebe15e0c767920c6ad4bffd39218d0ce79869ce8d55bc5d9c65e0e34cbf47af (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, RELEASE=wip-65480-squid-de5210d, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-15T16:07:04.973 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:04 smithi053 podman[101566]: 2024-05-15 16:07:04.971521714 +0000 UTC m=+0.443000925 container attach 2ebe15e0c767920c6ad4bffd39218d0ce79869ce8d55bc5d9c65e0e34cbf47af (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, RELEASE=wip-65480-squid-de5210d, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True) 2024-05-15T16:07:05.379 INFO:journalctl@ceph.osd.7.smithi053.stdout:May 15 16:07:05 smithi053 podman[101566]: 2024-05-15 16:07:05.195840573 +0000 UTC m=+0.667319789 container died 2ebe15e0c767920c6ad4bffd39218d0ce79869ce8d55bc5d9c65e0e34cbf47af (image=quay.ceph.io/ceph-ci/ceph@sha256:7cad1765acb37925dfdfc4273a170d55bf442c11104759fb310474235081d21a, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, RELEASE=wip-65480-squid-de5210d, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=) 2024-05-15T16:07:05.760 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@osd.7.service' 2024-05-15T16:07:05.804 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:07:05.805 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-05-15T16:07:05.805 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-05-15T16:07:05.805 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a 2024-05-15T16:07:06.129 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:05 smithi053 systemd[1]: Stopping Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:07:06.414 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.155Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-05-15T16:07:06.414 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.155Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-05-15T16:07:06.414 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.155Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-05-15T16:07:06.414 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.155Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-05-15T16:07:06.414 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.155Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-05-15T16:07:06.414 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.155Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-05-15T16:07:06.414 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.155Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-05-15T16:07:06.414 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.155Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-05-15T16:07:06.415 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.155Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-05-15T16:07:06.415 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.156Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-05-15T16:07:06.415 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.156Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-05-15T16:07:06.415 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a[63416]: ts=2024-05-15T16:07:06.156Z caller=main.go:1147 level=info msg="See you next time!" 2024-05-15T16:07:06.415 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 podman[101673]: 2024-05-15 16:07:06.165673439 +0000 UTC m=+0.129903555 container died c420668ab64015623c56a7ca304e621357a7d8821949c7352aae4a1ffbc392b6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T16:07:06.797 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 podman[101673]: 2024-05-15 16:07:06.594964663 +0000 UTC m=+0.559194781 container cleanup c420668ab64015623c56a7ca304e621357a7d8821949c7352aae4a1ffbc392b6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T16:07:06.797 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 bash[101673]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a 2024-05-15T16:07:07.045 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service' 2024-05-15T16:07:07.068 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:06 smithi053 podman[101685]: 2024-05-15 16:07:06.7961395 +0000 UTC m=+0.631930824 container remove c420668ab64015623c56a7ca304e621357a7d8821949c7352aae4a1ffbc392b6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-05-15T16:07:07.068 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:07 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-05-15T16:07:07.068 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:07 smithi053 systemd[1]: Stopped Ceph prometheus.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:07:07.068 INFO:journalctl@ceph.prometheus.a.smithi053.stdout:May 15 16:07:07 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@prometheus.a.service: Consumed 4.526s CPU time. 2024-05-15T16:07:07.081 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-15T16:07:07.081 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-05-15T16:07:07.081 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 22d82864-12cf-11ef-bc98-c7b262605968 --force --keep-logs 2024-05-15T16:07:08.899 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 16:07:08 smithi043 systemd[1]: Stopping Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:07:09.163 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 16:07:08 smithi043 podman[135204]: 2024-05-15 16:07:08.897419511 +0000 UTC m=+0.114433425 container died c92a2c2e9bf7b56b79f74b04fec996d6490f4950405e0361eb7e4d0d56226417 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T16:07:09.585 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 16:07:09 smithi043 podman[135204]: 2024-05-15 16:07:09.386703808 +0000 UTC m=+0.603717721 container cleanup c92a2c2e9bf7b56b79f74b04fec996d6490f4950405e0361eb7e4d0d56226417 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T16:07:09.585 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 16:07:09 smithi043 bash[135204]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a 2024-05-15T16:07:09.948 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 16:07:09 smithi043 podman[135216]: 2024-05-15 16:07:09.615372148 +0000 UTC m=+0.721565033 container remove c92a2c2e9bf7b56b79f74b04fec996d6490f4950405e0361eb7e4d0d56226417 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-05-15T16:07:09.949 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 16:07:09 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-05-15T16:07:09.949 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 16:07:09 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-05-15T16:07:09.949 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 16:07:09 smithi043 systemd[1]: Stopped Ceph node-exporter.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:07:09.949 INFO:journalctl@ceph.node-exporter.a.smithi043.stdout:May 15 16:07:09 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.a.service: Consumed 3.638s CPU time. 2024-05-15T16:07:10.235 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:09 smithi043 systemd[1]: Stopping Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:07:10.239 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:10 smithi043 ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a[76177]: ts=2024-05-15T16:07:10.193Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-05-15T16:07:10.552 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:10 smithi043 podman[135324]: 2024-05-15 16:07:10.234767765 +0000 UTC m=+0.140671329 container died bd365acb01c53c23c17d8acde42d75dadd6bc1006b78232554246bcdb10e9326 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T16:07:10.552 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:10 smithi043 podman[135324]: 2024-05-15 16:07:10.551641436 +0000 UTC m=+0.457544992 container cleanup bd365acb01c53c23c17d8acde42d75dadd6bc1006b78232554246bcdb10e9326 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T16:07:10.806 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:10 smithi043 bash[135324]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a 2024-05-15T16:07:10.806 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:10 smithi043 podman[135336]: 2024-05-15 16:07:10.744771168 +0000 UTC m=+0.513397581 container remove bd365acb01c53c23c17d8acde42d75dadd6bc1006b78232554246bcdb10e9326 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-05-15T16:07:10.806 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:10 smithi043 podman[135336]: 2024-05-15 16:07:10.803566776 +0000 UTC m=+0.572193144 volume remove f00a0c399f53f6b060c21d93bd25e957552360bc5584da4d430b758c8eec57d8 2024-05-15T16:07:11.059 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:11 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-05-15T16:07:11.059 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:11 smithi043 systemd[1]: Stopped Ceph alertmanager.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:07:11.059 INFO:journalctl@ceph.alertmanager.a.smithi043.stdout:May 15 16:07:11 smithi043 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@alertmanager.a.service: Consumed 1.729s CPU time. 2024-05-15T16:07:34.088 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 22d82864-12cf-11ef-bc98-c7b262605968 --force --keep-logs 2024-05-15T16:07:35.918 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 16:07:35 smithi053 systemd[1]: Stopping Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:07:36.215 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 16:07:35 smithi053 podman[102081]: 2024-05-15 16:07:35.933182144 +0000 UTC m=+0.109582326 container died 1fb31435f1519b91c0711eb92b1f9a7a0b90266b8453855b11a9e57dd7a9f0a8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T16:07:36.629 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 16:07:36 smithi053 podman[102081]: 2024-05-15 16:07:36.405723077 +0000 UTC m=+0.582123249 container cleanup 1fb31435f1519b91c0711eb92b1f9a7a0b90266b8453855b11a9e57dd7a9f0a8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T16:07:36.630 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 16:07:36 smithi053 bash[102081]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b 2024-05-15T16:07:37.046 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 16:07:36 smithi053 podman[102096]: 2024-05-15 16:07:36.665239064 +0000 UTC m=+0.730433351 container remove 1fb31435f1519b91c0711eb92b1f9a7a0b90266b8453855b11a9e57dd7a9f0a8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-05-15T16:07:37.047 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 16:07:36 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-05-15T16:07:37.047 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 16:07:36 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-05-15T16:07:37.047 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 16:07:36 smithi053 systemd[1]: Stopped Ceph node-exporter.b for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:07:37.047 INFO:journalctl@ceph.node-exporter.b.smithi053.stdout:May 15 16:07:36 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@node-exporter.b.service: Consumed 3.504s CPU time. 2024-05-15T16:07:37.303 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:37 smithi053 systemd[1]: Stopping Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968... 2024-05-15T16:07:37.629 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=server t=2024-05-15T16:07:37.301915555Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-05-15T16:07:37.630 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=tracing t=2024-05-15T16:07:37.302239269Z level=info msg="Closing tracing" 2024-05-15T16:07:37.630 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:37 smithi053 ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a[65623]: logger=ticker t=2024-05-15T16:07:37.302278498Z level=info msg=stopped last_tick=2024-05-15T16:07:30Z 2024-05-15T16:07:37.630 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:37 smithi053 podman[102203]: 2024-05-15 16:07:37.312444994 +0000 UTC m=+0.117710355 container died 16402042b8e562911a3ec6a45b26dbaacd937df0b6cb0f4f925fcd10cb85866d (image=quay.io/ceph/grafana:9.4.12, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-15T16:07:38.011 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:37 smithi053 podman[102203]: 2024-05-15 16:07:37.767773528 +0000 UTC m=+0.573038883 container cleanup 16402042b8e562911a3ec6a45b26dbaacd937df0b6cb0f4f925fcd10cb85866d (image=quay.io/ceph/grafana:9.4.12, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-15T16:07:38.011 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:37 smithi053 bash[102203]: ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a 2024-05-15T16:07:38.360 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:38 smithi053 podman[102217]: 2024-05-15 16:07:38.010552172 +0000 UTC m=+0.695675249 container remove 16402042b8e562911a3ec6a45b26dbaacd937df0b6cb0f4f925fcd10cb85866d (image=quay.io/ceph/grafana:9.4.12, name=ceph-22d82864-12cf-11ef-bc98-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-05-15T16:07:38.360 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:38 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@grafana.a.service: Deactivated successfully. 2024-05-15T16:07:38.360 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:38 smithi053 systemd[1]: Stopped Ceph grafana.a for 22d82864-12cf-11ef-bc98-c7b262605968. 2024-05-15T16:07:38.360 INFO:journalctl@ceph.grafana.a.smithi053.stdout:May 15 16:07:38 smithi053 systemd[1]: ceph-22d82864-12cf-11ef-bc98-c7b262605968@grafana.a.service: Consumed 11.503s CPU time. 2024-05-15T16:07:49.912 DEBUG:teuthology.orchestra.run.smithi043:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-15T16:07:49.949 DEBUG:teuthology.orchestra.run.smithi053:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-15T16:07:49.977 INFO:tasks.cephadm:Archiving crash dumps... 2024-05-15T16:07:49.979 DEBUG:teuthology.misc:Transferring archived files from smithi043:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165/remote/smithi043/crash 2024-05-15T16:07:49.980 DEBUG:teuthology.orchestra.run.smithi043:> sudo tar c -f - -C /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/crash -- . 2024-05-15T16:07:50.030 INFO:teuthology.orchestra.run.smithi043.stderr:tar: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/crash: Cannot open: No such file or directory 2024-05-15T16:07:50.031 INFO:teuthology.orchestra.run.smithi043.stderr:tar: Error is not recoverable: exiting now 2024-05-15T16:07:50.033 DEBUG:teuthology.misc:Transferring archived files from smithi053:/var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165/remote/smithi053/crash 2024-05-15T16:07:50.034 DEBUG:teuthology.orchestra.run.smithi053:> sudo tar c -f - -C /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/crash -- . 2024-05-15T16:07:50.078 INFO:teuthology.orchestra.run.smithi053.stderr:tar: /var/lib/ceph/22d82864-12cf-11ef-bc98-c7b262605968/crash: Cannot open: No such file or directory 2024-05-15T16:07:50.078 INFO:teuthology.orchestra.run.smithi053.stderr:tar: Error is not recoverable: exiting now 2024-05-15T16:07:50.079 INFO:tasks.cephadm:Checking cluster log for badness... 2024-05-15T16:07:50.079 DEBUG:teuthology.orchestra.run.smithi043:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-15T16:07:50.110 INFO:teuthology.orchestra.run.smithi043.stdout:2024-05-15T15:48:12.521986+0000 mon.a (mon.0) 1004 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T16:07:50.111 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-05-15T16:07:50.111 DEBUG:teuthology.orchestra.run.smithi043:> sudo egrep '\[SEC\]' /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-15T16:07:50.182 DEBUG:teuthology.orchestra.run.smithi043:> sudo egrep '\[ERR\]' /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-15T16:07:50.253 DEBUG:teuthology.orchestra.run.smithi043:> sudo egrep '\[WRN\]' /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-15T16:07:50.324 INFO:teuthology.orchestra.run.smithi043.stdout:2024-05-15T15:48:12.521986+0000 mon.a (mon.0) 1004 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-05-15T16:07:50.324 INFO:tasks.cephadm:Compressing logs... 2024-05-15T16:07:50.324 DEBUG:teuthology.orchestra.run.smithi043:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-15T16:07:50.367 DEBUG:teuthology.orchestra.run.smithi053:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-15T16:07:50.391 INFO:teuthology.orchestra.run.smithi043.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.log 2024-05-15T16:07:50.391 INFO:teuthology.orchestra.run.smithi043.stderr:: No such file or directory 2024-05-15T16:07:50.392 INFO:teuthology.orchestra.run.smithi043.stderr:gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.2.log 2024-05-15T16:07:50.393 INFO:teuthology.orchestra.run.smithi043.stderr:gzip/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.log: -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.audit.log 2024-05-15T16:07:50.393 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.3.log 2024-05-15T16:07:50.393 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.1.log 2024-05-15T16:07:50.393 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mgr.y.log 2024-05-15T16:07:50.400 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.0.log 2024-05-15T16:07:50.400 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mon.c.log 2024-05-15T16:07:50.400 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mon.a.log 2024-05-15T16:07:50.400 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mon.c.log: 93.8% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.log.gz 2024-05-15T16:07:50.402 INFO:teuthology.orchestra.run.smithi053.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-client.rgw.foo.smithi053.vjmwpu.log 2024-05-15T16:07:50.402 INFO:teuthology.orchestra.run.smithi053.stderr:: No such file or directory 2024-05-15T16:07:50.402 INFO:teuthology.orchestra.run.smithi053.stderr:gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.log 2024-05-15T16:07:50.402 INFO:teuthology.orchestra.run.smithi053.stderr:gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.audit.log 2024-05-15T16:07:50.402 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-client.rgw.foo.smithi053.vjmwpu.log: /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.log: 76.0% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-client.rgw.foo.smithi053.vjmwpu.log.gz 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.7.log 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.5.log 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.4.log 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mgr.x.log 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.6.log 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.cephadm.log 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-volume.log 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.cephadm.log: 89.0% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.log.gz 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mon.b.log 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-volume.log: 95.8% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.cephadm.log.gz 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-15T16:07:50.403 INFO:teuthology.orchestra.run.smithi053.stderr: 91.3% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.audit.log.gz 2024-05-15T16:07:50.405 INFO:teuthology.orchestra.run.smithi043.stderr: 94.5% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.audit.log.gz 2024-05-15T16:07:50.406 INFO:teuthology.orchestra.run.smithi043.stderr:gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.cephadm.log 2024-05-15T16:07:50.406 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-client.rgw.foo.smithi043.fiueec.log 2024-05-15T16:07:50.414 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.cephadm.log: 97.0% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph.cephadm.log.gz 2024-05-15T16:07:50.415 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 91.0% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-volume.log.gz 2024-05-15T16:07:50.419 INFO:teuthology.orchestra.run.smithi043.stderr:gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/tcmu-runner.log 2024-05-15T16:07:50.420 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-client.rgw.foo.smithi043.fiueec.log: 75.9% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-client.rgw.foo.smithi043.fiueec.log.gz 2024-05-15T16:07:50.432 INFO:teuthology.orchestra.run.smithi043.stderr:gzip -5 --verbose -- /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-volume.log 2024-05-15T16:07:50.433 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/tcmu-runner.log: 84.2% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/tcmu-runner.log.gz 2024-05-15T16:07:50.440 INFO:teuthology.orchestra.run.smithi043.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-15T16:07:50.448 INFO:teuthology.orchestra.run.smithi053.stderr: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-15T16:07:50.457 INFO:teuthology.orchestra.run.smithi043.stderr:/var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 90.9% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-volume.log.gz 2024-05-15T16:07:50.476 INFO:teuthology.orchestra.run.smithi043.stderr: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-15T16:07:50.549 INFO:teuthology.orchestra.run.smithi053.stderr: 89.9% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mgr.x.log.gz 2024-05-15T16:07:50.938 INFO:teuthology.orchestra.run.smithi043.stderr: 89.1% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mgr.y.log.gz 2024-05-15T16:07:51.118 INFO:teuthology.orchestra.run.smithi053.stderr: 92.3% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mon.b.log.gz 2024-05-15T16:07:51.247 INFO:teuthology.orchestra.run.smithi043.stderr: 92.4% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mon.c.log.gz 2024-05-15T16:07:51.828 INFO:teuthology.orchestra.run.smithi043.stderr: 93.3% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.2.log.gz 2024-05-15T16:07:51.904 INFO:teuthology.orchestra.run.smithi053.stderr: 93.2% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.6.log.gz 2024-05-15T16:07:52.176 INFO:teuthology.orchestra.run.smithi043.stderr: 91.0% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-mon.a.log.gz 2024-05-15T16:07:52.284 INFO:teuthology.orchestra.run.smithi043.stderr: 93.4% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.1.log.gz 2024-05-15T16:07:52.339 INFO:teuthology.orchestra.run.smithi053.stderr: 93.9% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.7.log.gz 2024-05-15T16:07:52.552 INFO:teuthology.orchestra.run.smithi053.stderr: 93.3% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.5.log.gz 2024-05-15T16:07:52.677 INFO:teuthology.orchestra.run.smithi043.stderr: 93.5% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.0.log.gz 2024-05-15T16:07:52.785 INFO:teuthology.orchestra.run.smithi053.stderr: 93.6% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.4.log.gz 2024-05-15T16:07:52.788 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-05-15T16:07:52.788 INFO:teuthology.orchestra.run.smithi053.stderr:real 0m2.405s 2024-05-15T16:07:52.788 INFO:teuthology.orchestra.run.smithi053.stderr:user 0m8.636s 2024-05-15T16:07:52.788 INFO:teuthology.orchestra.run.smithi053.stderr:sys 0m0.317s 2024-05-15T16:07:53.251 INFO:teuthology.orchestra.run.smithi043.stderr: 93.4% -- replaced with /var/log/ceph/22d82864-12cf-11ef-bc98-c7b262605968/ceph-osd.3.log.gz 2024-05-15T16:07:53.253 INFO:teuthology.orchestra.run.smithi043.stderr: 2024-05-15T16:07:53.253 INFO:teuthology.orchestra.run.smithi043.stderr:real 0m2.872s 2024-05-15T16:07:53.253 INFO:teuthology.orchestra.run.smithi043.stderr:user 0m11.310s 2024-05-15T16:07:53.253 INFO:teuthology.orchestra.run.smithi043.stderr:sys 0m0.393s 2024-05-15T16:07:53.253 INFO:tasks.cephadm:Archiving logs... 2024-05-15T16:07:53.253 DEBUG:teuthology.misc:Transferring archived files from smithi043:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165/remote/smithi043/log 2024-05-15T16:07:53.254 DEBUG:teuthology.orchestra.run.smithi043:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-15T16:07:54.817 DEBUG:teuthology.misc:Transferring archived files from smithi053:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165/remote/smithi053/log 2024-05-15T16:07:54.817 DEBUG:teuthology.orchestra.run.smithi053:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-15T16:07:56.064 INFO:tasks.cephadm:Removing cluster... 2024-05-15T16:07:56.064 DEBUG:teuthology.orchestra.run.smithi043:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 22d82864-12cf-11ef-bc98-c7b262605968 --force 2024-05-15T16:07:56.373 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 22d82864-12cf-11ef-bc98-c7b262605968 --force 2024-05-15T16:07:56.682 INFO:tasks.cephadm:Removing cephadm ... 2024-05-15T16:07:56.683 DEBUG:teuthology.orchestra.run.smithi043:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-15T16:07:56.700 DEBUG:teuthology.orchestra.run.smithi053:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-15T16:07:56.717 INFO:tasks.cephadm:Teardown complete 2024-05-15T16:07:56.717 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-05-15T16:07:56.729 INFO:teuthology.task.clock:Checking final clock skew... 2024-05-15T16:07:56.730 DEBUG:teuthology.orchestra.run.smithi043:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-15T16:07:56.743 DEBUG:teuthology.orchestra.run.smithi053:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-15T16:07:56.756 INFO:teuthology.orchestra.run.smithi043.stderr:bash: line 1: ntpq: command not found 2024-05-15T16:07:56.760 INFO:teuthology.orchestra.run.smithi043.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-15T16:07:56.760 INFO:teuthology.orchestra.run.smithi043.stdout:=============================================================================== 2024-05-15T16:07:56.761 INFO:teuthology.orchestra.run.smithi043.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 +580us[ +595us] +/- 61ms 2024-05-15T16:07:56.761 INFO:teuthology.orchestra.run.smithi043.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 54 -1650us[-1635us] +/- 62ms 2024-05-15T16:07:56.761 INFO:teuthology.orchestra.run.smithi043.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 54 +1230us[+1245us] +/- 71ms 2024-05-15T16:07:56.761 INFO:teuthology.orchestra.run.smithi043.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-15T16:07:56.771 INFO:teuthology.orchestra.run.smithi053.stderr:bash: line 1: ntpq: command not found 2024-05-15T16:07:56.776 INFO:teuthology.orchestra.run.smithi053.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-15T16:07:56.776 INFO:teuthology.orchestra.run.smithi053.stdout:=============================================================================== 2024-05-15T16:07:56.776 INFO:teuthology.orchestra.run.smithi053.stdout:^* hv01.front.sepia.ceph.com 3 6 377 49 +613us[ +607us] +/- 62ms 2024-05-15T16:07:56.776 INFO:teuthology.orchestra.run.smithi053.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 53 -1610us[-1616us] +/- 62ms 2024-05-15T16:07:56.776 INFO:teuthology.orchestra.run.smithi053.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 51 +1276us[+1269us] +/- 71ms 2024-05-15T16:07:56.776 INFO:teuthology.orchestra.run.smithi053.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-15T16:07:56.777 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-05-15T16:07:56.842 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-05-15T16:07:56.844 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-05-15T16:07:57.115 DEBUG:teuthology.orchestra.run.smithi043:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-15T16:07:57.214 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-15T16:07:57.295 DEBUG:teuthology.orchestra.run.smithi043:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-15T16:07:57.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-15T16:07:57.327 DEBUG:teuthology.orchestra.run.smithi053:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-05-15T16:07:57.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-15T16:07:57.358 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-05-15T16:07:57.368 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-05-15T16:07:57.516 INFO:teuthology.task.internal:Duration was 2915.384828 seconds 2024-05-15T16:07:57.517 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-05-15T16:07:57.661 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-05-15T16:07:57.662 DEBUG:teuthology.orchestra.run.smithi043:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-15T16:07:57.664 DEBUG:teuthology.orchestra.run.smithi053:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-15T16:07:57.709 INFO:teuthology.orchestra.run.smithi043.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-15T16:07:57.712 INFO:teuthology.orchestra.run.smithi053.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-15T16:07:58.176 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-05-15T16:07:58.176 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi043.front.sepia.ceph.com 2024-05-15T16:07:58.176 DEBUG:teuthology.orchestra.run.smithi043:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-15T16:07:58.201 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi053.front.sepia.ceph.com 2024-05-15T16:07:58.201 DEBUG:teuthology.orchestra.run.smithi053:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-15T16:07:58.227 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-05-15T16:07:58.227 DEBUG:teuthology.orchestra.run.smithi043:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-15T16:07:58.244 DEBUG:teuthology.orchestra.run.smithi053:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-15T16:07:58.297 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-05-15T16:07:58.298 DEBUG:teuthology.orchestra.run.smithi043:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-15T16:07:58.315 DEBUG:teuthology.orchestra.run.smithi053:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-15T16:07:59.002 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-05-15T16:07:59.012 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-05-15T16:07:59.013 DEBUG:teuthology.orchestra.run.smithi043:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-15T16:07:59.038 DEBUG:teuthology.orchestra.run.smithi053:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-15T16:07:59.064 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-05-15T16:07:59.237 DEBUG:teuthology.orchestra.run.smithi043:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-15T16:07:59.240 DEBUG:teuthology.orchestra.run.smithi053:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-15T16:07:59.262 INFO:teuthology.orchestra.run.smithi043.stdout:kernel.core_pattern = core 2024-05-15T16:07:59.265 INFO:teuthology.orchestra.run.smithi053.stdout:kernel.core_pattern = core 2024-05-15T16:07:59.289 DEBUG:teuthology.orchestra.run.smithi043:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-15T16:07:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-15T16:07:59.336 DEBUG:teuthology.orchestra.run.smithi053:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-15T16:07:59.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-15T16:07:59.353 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-05-15T16:07:59.368 INFO:teuthology.task.internal:Transferring archived files... 2024-05-15T16:07:59.369 DEBUG:teuthology.misc:Transferring archived files from smithi043:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165/remote/smithi043 2024-05-15T16:07:59.369 DEBUG:teuthology.orchestra.run.smithi043:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-15T16:07:59.434 DEBUG:teuthology.misc:Transferring archived files from smithi053:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165/remote/smithi053 2024-05-15T16:07:59.435 DEBUG:teuthology.orchestra.run.smithi053:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-15T16:07:59.478 INFO:teuthology.task.internal:Removing archive directory... 2024-05-15T16:07:59.479 DEBUG:teuthology.orchestra.run.smithi043:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-15T16:07:59.481 DEBUG:teuthology.orchestra.run.smithi053:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-15T16:07:59.528 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-05-15T16:07:59.540 INFO:teuthology.task.internal:Not uploading archives. 2024-05-15T16:07:59.540 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-05-15T16:07:59.582 INFO:teuthology.task.internal:Tidying up after the test... 2024-05-15T16:07:59.583 DEBUG:teuthology.orchestra.run.smithi043:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-15T16:07:59.585 DEBUG:teuthology.orchestra.run.smithi053:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-15T16:07:59.598 INFO:teuthology.orchestra.run.smithi043.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 15 16:07 /home/ubuntu/cephtest 2024-05-15T16:07:59.604 INFO:teuthology.orchestra.run.smithi053.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 15 16:07 /home/ubuntu/cephtest 2024-05-15T16:07:59.605 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-05-15T16:07:59.616 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-05-15T16:07:59.801 INFO:teuthology.nuke:Checking targets against current locks 2024-05-15T16:07:59.829 DEBUG:teuthology.nuke:shortname: smithi043 2024-05-15T16:07:59.830 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-15T16:07:59.853 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi043.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-15 15:11:03.052455', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEV/s1jhomw6GouUuRgv82hZgpb9i/dUDfIiPcDZ6e6p9Sm6A++kvvAuToPtje3gSPZk95DuIt1iFCeE+cblstY='} 2024-05-15T16:07:59.857 DEBUG:teuthology.nuke:shortname: smithi053 2024-05-15T16:07:59.858 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-15T16:07:59.884 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi053.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-05-15_15:01:21-orch:cephadm-wip-65480-squid-distro-default-smithi/7707165', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-15 15:11:03.050486', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNW37ucWkq64eR0bVE4G0GhITfumYcKeYmmwMjxzF41yJvjbxq+X+l6WWaq8t8dD58kOevOQU9kV5mow7L3cPZI='} 2024-05-15T16:07:59.910 INFO:teuthology.orchestra.console.smithi043:Power off 2024-05-15T16:07:59.911 DEBUG:teuthology.orchestra.console.smithi043:pexpect command: ipmitool -H smithi043.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-05-15T16:07:59.937 DEBUG:teuthology.orchestra.console.smithi043:power off output: Chassis Power Control: Down/Off 2024-05-15T16:07:59.937 DEBUG:teuthology.orchestra.console.smithi043:pexpect command: ipmitool -H smithi043.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-15T16:07:59.957 INFO:teuthology.orchestra.console.smithi053:Power off 2024-05-15T16:07:59.958 DEBUG:teuthology.orchestra.console.smithi053:pexpect command: ipmitool -H smithi053.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-05-15T16:07:59.967 DEBUG:teuthology.orchestra.console.smithi043:check power output: Chassis Power is on 2024-05-15T16:07:59.982 DEBUG:teuthology.orchestra.console.smithi053:power off output: Chassis Power Control: Down/Off 2024-05-15T16:07:59.982 DEBUG:teuthology.orchestra.console.smithi053:pexpect command: ipmitool -H smithi053.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-15T16:08:00.005 DEBUG:teuthology.orchestra.console.smithi053:check power output: Chassis Power is on 2024-05-15T16:08:03.969 DEBUG:teuthology.orchestra.console.smithi043:pexpect command: ipmitool -H smithi043.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-15T16:08:04.007 DEBUG:teuthology.orchestra.console.smithi053:pexpect command: ipmitool -H smithi053.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-15T16:08:04.081 DEBUG:teuthology.orchestra.console.smithi043:check power output: Chassis Power is on 2024-05-15T16:08:04.118 DEBUG:teuthology.orchestra.console.smithi053:check power output: Chassis Power is on 2024-05-15T16:08:08.086 DEBUG:teuthology.orchestra.console.smithi043:pexpect command: ipmitool -H smithi043.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-15T16:08:08.120 DEBUG:teuthology.orchestra.console.smithi053:pexpect command: ipmitool -H smithi053.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-15T16:08:08.197 DEBUG:teuthology.orchestra.console.smithi043:check power output: Chassis Power is off 2024-05-15T16:08:08.230 DEBUG:teuthology.orchestra.console.smithi053:check power output: Chassis Power is off 2024-05-15T16:08:08.298 INFO:teuthology.orchestra.console.smithi043:Power off completed 2024-05-15T16:08:08.331 INFO:teuthology.orchestra.console.smithi053:Power off completed 2024-05-15T16:08:08.435 INFO:teuthology.run:Summary data: description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2915.3848276138306 failure_reason: '"2024-05-15T15:48:12.521986+0000 mon.a (mon.0) 1004 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_gabrioux@teuthology success: false 2024-05-15T16:08:08.435 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-15T16:08:08.538 INFO:teuthology.run:FAIL